summaryrefslogtreecommitdiff
path: root/drivers/staging/zcache
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/staging/zcache')
-rw-r--r--drivers/staging/zcache/Kconfig18
-rw-r--r--drivers/staging/zcache/Makefile2
-rw-r--r--drivers/staging/zcache/TODO9
-rw-r--r--drivers/staging/zcache/debug.c107
-rw-r--r--drivers/staging/zcache/debug.h305
-rw-r--r--drivers/staging/zcache/ramster/debug.c66
-rw-r--r--drivers/staging/zcache/ramster/debug.h145
-rw-r--r--drivers/staging/zcache/ramster/ramster.c141
-rw-r--r--drivers/staging/zcache/ramster/tcp.c25
-rw-r--r--drivers/staging/zcache/zbud.c7
-rw-r--r--drivers/staging/zcache/zcache-main.c526
11 files changed, 892 insertions, 459 deletions
diff --git a/drivers/staging/zcache/Kconfig b/drivers/staging/zcache/Kconfig
index 73582705e8c5..05e87a1e5d93 100644
--- a/drivers/staging/zcache/Kconfig
+++ b/drivers/staging/zcache/Kconfig
@@ -10,12 +10,20 @@ config ZCACHE
memory to store clean page cache pages and swap in RAM,
providing a noticeable reduction in disk I/O.
+config ZCACHE_DEBUG
+ bool "Enable debug statistics"
+ depends on DEBUG_FS && ZCACHE
+ default n
+ help
+ This is used to provide an debugfs directory with counters of
+ how zcache is doing. You probably want to set this to 'N'.
+
config RAMSTER
bool "Cross-machine RAM capacity sharing, aka peer-to-peer tmem"
depends on CONFIGFS_FS=y && SYSFS=y && !HIGHMEM && ZCACHE=y
depends on NET
# must ensure struct page is 8-byte aligned
- select HAVE_ALIGNED_STRUCT_PAGE if !64_BIT
+ select HAVE_ALIGNED_STRUCT_PAGE if !64BIT
default n
help
RAMster allows RAM on other machines in a cluster to be utilized
@@ -25,6 +33,14 @@ config RAMSTER
zcache2, compresses swap pages into local RAM, but then remotifies
the compressed pages to another node in the RAMster cluster.
+config RAMSTER_DEBUG
+ bool "Enable ramster debug statistics"
+ depends on DEBUG_FS && RAMSTER
+ default n
+ help
+ This is used to provide an debugfs directory with counters of
+ how ramster is doing. You probably want to set this to 'N'.
+
# Depends on not-yet-upstreamed mm patches to export end_swap_bio_write and
# __add_to_swap_cache, and implement __swap_writepage (which is swap_writepage
# without the frontswap call. When these are in-tree, the dependency on
diff --git a/drivers/staging/zcache/Makefile b/drivers/staging/zcache/Makefile
index 471104957dad..845a5c2721b5 100644
--- a/drivers/staging/zcache/Makefile
+++ b/drivers/staging/zcache/Makefile
@@ -1,4 +1,6 @@
zcache-y := zcache-main.o tmem.o zbud.o
+zcache-$(CONFIG_ZCACHE_DEBUG) += debug.o
+zcache-$(CONFIG_RAMSTER_DEBUG) += ramster/debug.o
zcache-$(CONFIG_RAMSTER) += ramster/ramster.o ramster/r2net.o
zcache-$(CONFIG_RAMSTER) += ramster/nodemanager.o ramster/tcp.o
zcache-$(CONFIG_RAMSTER) += ramster/heartbeat.o ramster/masklog.o
diff --git a/drivers/staging/zcache/TODO b/drivers/staging/zcache/TODO
index c1e26d4973dc..d0c18fa95743 100644
--- a/drivers/staging/zcache/TODO
+++ b/drivers/staging/zcache/TODO
@@ -41,14 +41,10 @@ STATUS/OWNERSHIP
for 3.9, see https://lkml.org/lkml/2013/2/6/437;
7. PROTOTYPED as part of "new" zcache; in staging/zcache for 3.9;
needs more review (plan to discuss at LSF/MM 2013)
-8. IN PROGRESS; owned by Konrad Wilk; v2 recently posted
- http://lkml.org/lkml/2013/2/1/542
9. IN PROGRESS; owned by Konrad Wilk; Mel Gorman provided
great feedback in August 2012 (unfortunately of "old"
zcache)
-10. Konrad posted series of fixes (that now need rebasing)
- https://lkml.org/lkml/2013/2/1/566
-11. NOT DONE; owned by Konrad Wilk
+11. NOT DONE; owned by Konrad Wilk and Bob Liu
12. TBD (depends on quantity of feedback)
13. PROPOSED; one suggestion proposed by Dan; needs more ideas/feedback
14. TBD (depends on feedback)
@@ -65,5 +61,4 @@ ZCACHE FUTURE NEW FUNCTIONALITY
A. Support zsmalloc as an alternative high-density allocator
(See https://lkml.org/lkml/2013/1/23/511)
-B. Support zero-filled pages more efficiently
-C. Possibly support three zbuds per pageframe when space allows
+B. Possibly support three zbuds per pageframe when space allows
diff --git a/drivers/staging/zcache/debug.c b/drivers/staging/zcache/debug.c
new file mode 100644
index 000000000000..daa26919b886
--- /dev/null
+++ b/drivers/staging/zcache/debug.c
@@ -0,0 +1,107 @@
+#include <linux/atomic.h>
+#include "debug.h"
+
+#ifdef CONFIG_ZCACHE_DEBUG
+#include <linux/debugfs.h>
+
+ssize_t zcache_obj_count;
+ssize_t zcache_obj_count_max;
+ssize_t zcache_objnode_count;
+ssize_t zcache_objnode_count_max;
+u64 zcache_eph_zbytes;
+u64 zcache_eph_zbytes_max;
+u64 zcache_pers_zbytes_max;
+ssize_t zcache_eph_pageframes_max;
+ssize_t zcache_pers_pageframes_max;
+ssize_t zcache_pageframes_alloced;
+ssize_t zcache_pageframes_freed;
+ssize_t zcache_eph_zpages;
+ssize_t zcache_eph_zpages_max;
+ssize_t zcache_pers_zpages_max;
+ssize_t zcache_flush_total;
+ssize_t zcache_flush_found;
+ssize_t zcache_flobj_total;
+ssize_t zcache_flobj_found;
+ssize_t zcache_failed_eph_puts;
+ssize_t zcache_failed_pers_puts;
+ssize_t zcache_failed_getfreepages;
+ssize_t zcache_failed_alloc;
+ssize_t zcache_put_to_flush;
+ssize_t zcache_compress_poor;
+ssize_t zcache_mean_compress_poor;
+ssize_t zcache_eph_ate_tail;
+ssize_t zcache_eph_ate_tail_failed;
+ssize_t zcache_pers_ate_eph;
+ssize_t zcache_pers_ate_eph_failed;
+ssize_t zcache_evicted_eph_zpages;
+ssize_t zcache_evicted_eph_pageframes;
+ssize_t zcache_zero_filled_pages;
+ssize_t zcache_zero_filled_pages_max;
+
+#define ATTR(x) { .name = #x, .val = &zcache_##x, }
+static struct debug_entry {
+ const char *name;
+ ssize_t *val;
+} attrs[] = {
+ ATTR(obj_count), ATTR(obj_count_max),
+ ATTR(objnode_count), ATTR(objnode_count_max),
+ ATTR(flush_total), ATTR(flush_found),
+ ATTR(flobj_total), ATTR(flobj_found),
+ ATTR(failed_eph_puts), ATTR(failed_pers_puts),
+ ATTR(failed_getfreepages), ATTR(failed_alloc),
+ ATTR(put_to_flush),
+ ATTR(compress_poor), ATTR(mean_compress_poor),
+ ATTR(eph_ate_tail), ATTR(eph_ate_tail_failed),
+ ATTR(pers_ate_eph), ATTR(pers_ate_eph_failed),
+ ATTR(evicted_eph_zpages), ATTR(evicted_eph_pageframes),
+ ATTR(eph_pageframes), ATTR(eph_pageframes_max),
+ ATTR(pers_pageframes), ATTR(pers_pageframes_max),
+ ATTR(eph_zpages), ATTR(eph_zpages_max),
+ ATTR(pers_zpages), ATTR(pers_zpages_max),
+ ATTR(last_active_file_pageframes),
+ ATTR(last_inactive_file_pageframes),
+ ATTR(last_active_anon_pageframes),
+ ATTR(last_inactive_anon_pageframes),
+ ATTR(eph_nonactive_puts_ignored),
+ ATTR(pers_nonactive_puts_ignored),
+ ATTR(zero_filled_pages),
+#ifdef CONFIG_ZCACHE_WRITEBACK
+ ATTR(outstanding_writeback_pages),
+ ATTR(writtenback_pages),
+#endif
+};
+#undef ATTR
+int zcache_debugfs_init(void)
+{
+ unsigned int i;
+ struct dentry *root = debugfs_create_dir("zcache", NULL);
+ if (root == NULL)
+ return -ENXIO;
+
+ for (i = 0; i < ARRAY_SIZE(attrs); i++)
+ if (!debugfs_create_size_t(attrs[i].name, S_IRUGO, root, attrs[i].val))
+ goto out;
+
+ debugfs_create_u64("eph_zbytes", S_IRUGO, root, &zcache_eph_zbytes);
+ debugfs_create_u64("eph_zbytes_max", S_IRUGO, root, &zcache_eph_zbytes_max);
+ debugfs_create_u64("pers_zbytes", S_IRUGO, root, &zcache_pers_zbytes);
+ debugfs_create_u64("pers_zbytes_max", S_IRUGO, root, &zcache_pers_zbytes_max);
+
+ return 0;
+out:
+ return -ENODEV;
+}
+
+/* developers can call this in case of ooms, e.g. to find memory leaks */
+void zcache_dump(void)
+{
+ unsigned int i;
+ for (i = 0; i < ARRAY_SIZE(attrs); i++)
+ pr_debug("zcache: %s=%zu\n", attrs[i].name, *attrs[i].val);
+
+ pr_debug("zcache: eph_zbytes=%llu\n", (unsigned long long)zcache_eph_zbytes);
+ pr_debug("zcache: eph_zbytes_max=%llu\n", (unsigned long long)zcache_eph_zbytes_max);
+ pr_debug("zcache: pers_zbytes=%llu\n", (unsigned long long)zcache_pers_zbytes);
+ pr_debug("zcache: pers_zbytes_max=%llu\n", (unsigned long long)zcache_pers_zbytes_max);
+}
+#endif
diff --git a/drivers/staging/zcache/debug.h b/drivers/staging/zcache/debug.h
new file mode 100644
index 000000000000..8088d28f2dce
--- /dev/null
+++ b/drivers/staging/zcache/debug.h
@@ -0,0 +1,305 @@
+#include <linux/bug.h>
+
+#ifdef CONFIG_ZCACHE_DEBUG
+
+/* we try to keep these statistics SMP-consistent */
+extern ssize_t zcache_obj_count;
+static atomic_t zcache_obj_atomic = ATOMIC_INIT(0);
+extern ssize_t zcache_obj_count_max;
+static inline void inc_zcache_obj_count(void)
+{
+ zcache_obj_count = atomic_inc_return(&zcache_obj_atomic);
+ if (zcache_obj_count > zcache_obj_count_max)
+ zcache_obj_count_max = zcache_obj_count;
+}
+static inline void dec_zcache_obj_count(void)
+{
+ zcache_obj_count = atomic_dec_return(&zcache_obj_atomic);
+ BUG_ON(zcache_obj_count < 0);
+};
+extern ssize_t zcache_objnode_count;
+static atomic_t zcache_objnode_atomic = ATOMIC_INIT(0);
+extern ssize_t zcache_objnode_count_max;
+static inline void inc_zcache_objnode_count(void)
+{
+ zcache_objnode_count = atomic_inc_return(&zcache_objnode_atomic);
+ if (zcache_objnode_count > zcache_objnode_count_max)
+ zcache_objnode_count_max = zcache_objnode_count;
+};
+static inline void dec_zcache_objnode_count(void)
+{
+ zcache_objnode_count = atomic_dec_return(&zcache_objnode_atomic);
+ BUG_ON(zcache_objnode_count < 0);
+};
+extern u64 zcache_eph_zbytes;
+static atomic_long_t zcache_eph_zbytes_atomic = ATOMIC_INIT(0);
+extern u64 zcache_eph_zbytes_max;
+static inline void inc_zcache_eph_zbytes(unsigned clen)
+{
+ zcache_eph_zbytes = atomic_long_add_return(clen, &zcache_eph_zbytes_atomic);
+ if (zcache_eph_zbytes > zcache_eph_zbytes_max)
+ zcache_eph_zbytes_max = zcache_eph_zbytes;
+};
+static inline void dec_zcache_eph_zbytes(unsigned zsize)
+{
+ zcache_eph_zbytes = atomic_long_sub_return(zsize, &zcache_eph_zbytes_atomic);
+};
+extern u64 zcache_pers_zbytes;
+static atomic_long_t zcache_pers_zbytes_atomic = ATOMIC_INIT(0);
+extern u64 zcache_pers_zbytes_max;
+static inline void inc_zcache_pers_zbytes(unsigned clen)
+{
+ zcache_pers_zbytes = atomic_long_add_return(clen, &zcache_pers_zbytes_atomic);
+ if (zcache_pers_zbytes > zcache_pers_zbytes_max)
+ zcache_pers_zbytes_max = zcache_pers_zbytes;
+}
+static inline void dec_zcache_pers_zbytes(unsigned zsize)
+{
+ zcache_pers_zbytes = atomic_long_sub_return(zsize, &zcache_pers_zbytes_atomic);
+}
+extern ssize_t zcache_eph_pageframes;
+static atomic_t zcache_eph_pageframes_atomic = ATOMIC_INIT(0);
+extern ssize_t zcache_eph_pageframes_max;
+static inline void inc_zcache_eph_pageframes(void)
+{
+ zcache_eph_pageframes = atomic_inc_return(&zcache_eph_pageframes_atomic);
+ if (zcache_eph_pageframes > zcache_eph_pageframes_max)
+ zcache_eph_pageframes_max = zcache_eph_pageframes;
+};
+static inline void dec_zcache_eph_pageframes(void)
+{
+ zcache_eph_pageframes = atomic_dec_return(&zcache_eph_pageframes_atomic);
+};
+extern ssize_t zcache_pers_pageframes;
+static atomic_t zcache_pers_pageframes_atomic = ATOMIC_INIT(0);
+extern ssize_t zcache_pers_pageframes_max;
+static inline void inc_zcache_pers_pageframes(void)
+{
+ zcache_pers_pageframes = atomic_inc_return(&zcache_pers_pageframes_atomic);
+ if (zcache_pers_pageframes > zcache_pers_pageframes_max)
+ zcache_pers_pageframes_max = zcache_pers_pageframes;
+}
+static inline void dec_zcache_pers_pageframes(void)
+{
+ zcache_pers_pageframes = atomic_dec_return(&zcache_pers_pageframes_atomic);
+}
+extern ssize_t zcache_pageframes_alloced;
+static atomic_t zcache_pageframes_alloced_atomic = ATOMIC_INIT(0);
+static inline void inc_zcache_pageframes_alloced(void)
+{
+ zcache_pageframes_alloced = atomic_inc_return(&zcache_pageframes_alloced_atomic);
+};
+extern ssize_t zcache_pageframes_freed;
+static atomic_t zcache_pageframes_freed_atomic = ATOMIC_INIT(0);
+static inline void inc_zcache_pageframes_freed(void)
+{
+ zcache_pageframes_freed = atomic_inc_return(&zcache_pageframes_freed_atomic);
+}
+extern ssize_t zcache_eph_zpages;
+static atomic_t zcache_eph_zpages_atomic = ATOMIC_INIT(0);
+extern ssize_t zcache_eph_zpages_max;
+static inline void inc_zcache_eph_zpages(void)
+{
+ zcache_eph_zpages = atomic_inc_return(&zcache_eph_zpages_atomic);
+ if (zcache_eph_zpages > zcache_eph_zpages_max)
+ zcache_eph_zpages_max = zcache_eph_zpages;
+}
+static inline void dec_zcache_eph_zpages(unsigned zpages)
+{
+ zcache_eph_zpages = atomic_sub_return(zpages, &zcache_eph_zpages_atomic);
+}
+extern ssize_t zcache_pers_zpages;
+static atomic_t zcache_pers_zpages_atomic = ATOMIC_INIT(0);
+extern ssize_t zcache_pers_zpages_max;
+static inline void inc_zcache_pers_zpages(void)
+{
+ zcache_pers_zpages = atomic_inc_return(&zcache_pers_zpages_atomic);
+ if (zcache_pers_zpages > zcache_pers_zpages_max)
+ zcache_pers_zpages_max = zcache_pers_zpages;
+}
+static inline void dec_zcache_pers_zpages(unsigned zpages)
+{
+ zcache_pers_zpages = atomic_sub_return(zpages, &zcache_pers_zpages_atomic);
+}
+
+extern ssize_t zcache_zero_filled_pages;
+static atomic_t zcache_zero_filled_pages_atomic = ATOMIC_INIT(0);
+extern ssize_t zcache_zero_filled_pages_max;
+static inline void inc_zcache_zero_filled_pages(void)
+{
+ zcache_zero_filled_pages = atomic_inc_return(
+ &zcache_zero_filled_pages_atomic);
+ if (zcache_zero_filled_pages > zcache_zero_filled_pages_max)
+ zcache_zero_filled_pages_max = zcache_zero_filled_pages;
+}
+static inline void dec_zcache_zero_filled_pages(void)
+{
+ zcache_zero_filled_pages = atomic_dec_return(
+ &zcache_zero_filled_pages_atomic);
+}
+static inline unsigned long curr_pageframes_count(void)
+{
+ return zcache_pageframes_alloced -
+ atomic_read(&zcache_pageframes_freed_atomic) -
+ atomic_read(&zcache_eph_pageframes_atomic) -
+ atomic_read(&zcache_pers_pageframes_atomic);
+};
+/* but for the rest of these, counting races are ok */
+extern ssize_t zcache_flush_total;
+extern ssize_t zcache_flush_found;
+extern ssize_t zcache_flobj_total;
+extern ssize_t zcache_flobj_found;
+extern ssize_t zcache_failed_eph_puts;
+extern ssize_t zcache_failed_pers_puts;
+extern ssize_t zcache_failed_getfreepages;
+extern ssize_t zcache_failed_alloc;
+extern ssize_t zcache_put_to_flush;
+extern ssize_t zcache_compress_poor;
+extern ssize_t zcache_mean_compress_poor;
+extern ssize_t zcache_eph_ate_tail;
+extern ssize_t zcache_eph_ate_tail_failed;
+extern ssize_t zcache_pers_ate_eph;
+extern ssize_t zcache_pers_ate_eph_failed;
+extern ssize_t zcache_evicted_eph_zpages;
+extern ssize_t zcache_evicted_eph_pageframes;
+
+extern ssize_t zcache_last_active_file_pageframes;
+extern ssize_t zcache_last_inactive_file_pageframes;
+extern ssize_t zcache_last_active_anon_pageframes;
+extern ssize_t zcache_last_inactive_anon_pageframes;
+static ssize_t zcache_eph_nonactive_puts_ignored;
+static ssize_t zcache_pers_nonactive_puts_ignored;
+#ifdef CONFIG_ZCACHE_WRITEBACK
+extern ssize_t zcache_writtenback_pages;
+extern ssize_t zcache_outstanding_writeback_pages;
+#endif
+
+static inline void inc_zcache_flush_total(void)
+{
+ zcache_flush_total++;
+};
+static inline void inc_zcache_flush_found(void)
+{
+ zcache_flush_found++;
+};
+static inline void inc_zcache_flobj_total(void)
+{
+ zcache_flobj_total++;
+};
+static inline void inc_zcache_flobj_found(void)
+{
+ zcache_flobj_found++;
+};
+static inline void inc_zcache_failed_eph_puts(void)
+{
+ zcache_failed_eph_puts++;
+};
+static inline void inc_zcache_failed_pers_puts(void)
+{
+ zcache_failed_pers_puts++;
+};
+static inline void inc_zcache_failed_getfreepages(void)
+{
+ zcache_failed_getfreepages++;
+};
+static inline void inc_zcache_failed_alloc(void)
+{
+ zcache_failed_alloc++;
+};
+static inline void inc_zcache_put_to_flush(void)
+{
+ zcache_put_to_flush++;
+};
+static inline void inc_zcache_compress_poor(void)
+{
+ zcache_compress_poor++;
+};
+static inline void inc_zcache_mean_compress_poor(void)
+{
+ zcache_mean_compress_poor++;
+};
+static inline void inc_zcache_eph_ate_tail(void)
+{
+ zcache_eph_ate_tail++;
+};
+static inline void inc_zcache_eph_ate_tail_failed(void)
+{
+ zcache_eph_ate_tail_failed++;
+};
+static inline void inc_zcache_pers_ate_eph(void)
+{
+ zcache_pers_ate_eph++;
+};
+static inline void inc_zcache_pers_ate_eph_failed(void)
+{
+ zcache_pers_ate_eph_failed++;
+};
+static inline void inc_zcache_evicted_eph_zpages(unsigned zpages)
+{
+ zcache_evicted_eph_zpages += zpages;
+};
+static inline void inc_zcache_evicted_eph_pageframes(void)
+{
+ zcache_evicted_eph_pageframes++;
+};
+
+static inline void inc_zcache_eph_nonactive_puts_ignored(void)
+{
+ zcache_eph_nonactive_puts_ignored++;
+};
+static inline void inc_zcache_pers_nonactive_puts_ignored(void)
+{
+ zcache_pers_nonactive_puts_ignored++;
+};
+
+int zcache_debugfs_init(void);
+#else
+static inline void inc_zcache_obj_count(void) { };
+static inline void dec_zcache_obj_count(void) { };
+static inline void inc_zcache_objnode_count(void) { };
+static inline void dec_zcache_objnode_count(void) { };
+static inline void inc_zcache_eph_zbytes(unsigned clen) { };
+static inline void dec_zcache_eph_zbytes(unsigned zsize) { };
+static inline void inc_zcache_pers_zbytes(unsigned clen) { };
+static inline void dec_zcache_pers_zbytes(unsigned zsize) { };
+static inline void inc_zcache_eph_pageframes(void) { };
+static inline void dec_zcache_eph_pageframes(void) { };
+static inline void inc_zcache_pers_pageframes(void) { };
+static inline void dec_zcache_pers_pageframes(void) { };
+static inline void inc_zcache_pageframes_alloced(void) { };
+static inline void inc_zcache_pageframes_freed(void) { };
+static inline void inc_zcache_eph_zpages(void) { };
+static inline void dec_zcache_eph_zpages(unsigned zpages) { };
+static inline void inc_zcache_pers_zpages(void) { };
+static inline void dec_zcache_pers_zpages(unsigned zpages) { };
+static inline void inc_zcache_zero_filled_pages(void) { };
+static inline void dec_zcache_zero_filled_pages(void) { };
+static inline unsigned long curr_pageframes_count(void)
+{
+ return 0;
+};
+static inline int zcache_debugfs_init(void)
+{
+ return 0;
+};
+static inline void inc_zcache_flush_total(void) { };
+static inline void inc_zcache_flush_found(void) { };
+static inline void inc_zcache_flobj_total(void) { };
+static inline void inc_zcache_flobj_found(void) { };
+static inline void inc_zcache_failed_eph_puts(void) { };
+static inline void inc_zcache_failed_pers_puts(void) { };
+static inline void inc_zcache_failed_getfreepages(void) { };
+static inline void inc_zcache_failed_alloc(void) { };
+static inline void inc_zcache_put_to_flush(void) { };
+static inline void inc_zcache_compress_poor(void) { };
+static inline void inc_zcache_mean_compress_poor(void) { };
+static inline void inc_zcache_eph_ate_tail(void) { };
+static inline void inc_zcache_eph_ate_tail_failed(void) { };
+static inline void inc_zcache_pers_ate_eph(void) { };
+static inline void inc_zcache_pers_ate_eph_failed(void) { };
+static inline void inc_zcache_evicted_eph_zpages(unsigned zpages) { };
+static inline void inc_zcache_evicted_eph_pageframes(void) { };
+
+static inline void inc_zcache_eph_nonactive_puts_ignored(void) { };
+static inline void inc_zcache_pers_nonactive_puts_ignored(void) { };
+#endif
diff --git a/drivers/staging/zcache/ramster/debug.c b/drivers/staging/zcache/ramster/debug.c
new file mode 100644
index 000000000000..bf34133cc631
--- /dev/null
+++ b/drivers/staging/zcache/ramster/debug.c
@@ -0,0 +1,66 @@
+#include <linux/atomic.h>
+#include "debug.h"
+
+#ifdef CONFIG_DEBUG_FS
+#include <linux/debugfs.h>
+
+ssize_t ramster_eph_pages_remoted;
+ssize_t ramster_pers_pages_remoted;
+ssize_t ramster_eph_pages_remote_failed;
+ssize_t ramster_pers_pages_remote_failed;
+ssize_t ramster_remote_eph_pages_succ_get;
+ssize_t ramster_remote_pers_pages_succ_get;
+ssize_t ramster_remote_eph_pages_unsucc_get;
+ssize_t ramster_remote_pers_pages_unsucc_get;
+ssize_t ramster_pers_pages_remote_nomem;
+ssize_t ramster_remote_objects_flushed;
+ssize_t ramster_remote_object_flushes_failed;
+ssize_t ramster_remote_pages_flushed;
+ssize_t ramster_remote_page_flushes_failed;
+
+#define ATTR(x) { .name = #x, .val = &ramster_##x, }
+static struct debug_entry {
+ const char *name;
+ ssize_t *val;
+} attrs[] = {
+ ATTR(eph_pages_remoted),
+ ATTR(pers_pages_remoted),
+ ATTR(eph_pages_remote_failed),
+ ATTR(pers_pages_remote_failed),
+ ATTR(remote_eph_pages_succ_get),
+ ATTR(remote_pers_pages_succ_get),
+ ATTR(remote_eph_pages_unsucc_get),
+ ATTR(remote_pers_pages_unsucc_get),
+ ATTR(pers_pages_remote_nomem),
+ ATTR(remote_objects_flushed),
+ ATTR(remote_pages_flushed),
+ ATTR(remote_object_flushes_failed),
+ ATTR(remote_page_flushes_failed),
+ ATTR(foreign_eph_pages),
+ ATTR(foreign_eph_pages_max),
+ ATTR(foreign_pers_pages),
+ ATTR(foreign_pers_pages_max),
+};
+#undef ATTR
+
+int __init ramster_debugfs_init(void)
+{
+ int i;
+ struct dentry *root = debugfs_create_dir("ramster", NULL);
+ if (root == NULL)
+ return -ENXIO;
+
+ for (i = 0; i < ARRAY_SIZE(attrs); i++)
+ if (!debugfs_create_size_t(attrs[i].name,
+ S_IRUGO, root, attrs[i].val))
+ goto out;
+ return 0;
+out:
+ return -ENODEV;
+}
+#else
+static inline int ramster_debugfs_init(void)
+{
+ return 0;
+}
+#endif
diff --git a/drivers/staging/zcache/ramster/debug.h b/drivers/staging/zcache/ramster/debug.h
new file mode 100644
index 000000000000..5ffab50807d8
--- /dev/null
+++ b/drivers/staging/zcache/ramster/debug.h
@@ -0,0 +1,145 @@
+#include <linux/bug.h>
+
+#ifdef CONFIG_RAMSTER_DEBUG
+
+extern long ramster_flnodes;
+static atomic_t ramster_flnodes_atomic = ATOMIC_INIT(0);
+static unsigned long ramster_flnodes_max;
+static inline void inc_ramster_flnodes(void)
+{
+ ramster_flnodes = atomic_inc_return(&ramster_flnodes_atomic);
+ if (ramster_flnodes > ramster_flnodes_max)
+ ramster_flnodes_max = ramster_flnodes;
+}
+static inline void dec_ramster_flnodes(void)
+{
+ ramster_flnodes = atomic_dec_return(&ramster_flnodes_atomic);
+}
+extern ssize_t ramster_foreign_eph_pages;
+static atomic_t ramster_foreign_eph_pages_atomic = ATOMIC_INIT(0);
+static ssize_t ramster_foreign_eph_pages_max;
+static inline void inc_ramster_foreign_eph_pages(void)
+{
+ ramster_foreign_eph_pages = atomic_inc_return(
+ &ramster_foreign_eph_pages_atomic);
+ if (ramster_foreign_eph_pages > ramster_foreign_eph_pages_max)
+ ramster_foreign_eph_pages_max = ramster_foreign_eph_pages;
+}
+static inline void dec_ramster_foreign_eph_pages(void)
+{
+ ramster_foreign_eph_pages = atomic_dec_return(
+ &ramster_foreign_eph_pages_atomic);
+}
+extern ssize_t ramster_foreign_pers_pages;
+static atomic_t ramster_foreign_pers_pages_atomic = ATOMIC_INIT(0);
+static ssize_t ramster_foreign_pers_pages_max;
+static inline void inc_ramster_foreign_pers_pages(void)
+{
+ ramster_foreign_pers_pages = atomic_inc_return(
+ &ramster_foreign_pers_pages_atomic);
+ if (ramster_foreign_pers_pages > ramster_foreign_pers_pages_max)
+ ramster_foreign_pers_pages_max = ramster_foreign_pers_pages;
+}
+static inline void dec_ramster_foreign_pers_pages(void)
+{
+ ramster_foreign_pers_pages = atomic_dec_return(
+ &ramster_foreign_pers_pages_atomic);
+}
+
+extern ssize_t ramster_eph_pages_remoted;
+extern ssize_t ramster_pers_pages_remoted;
+extern ssize_t ramster_eph_pages_remote_failed;
+extern ssize_t ramster_pers_pages_remote_failed;
+extern ssize_t ramster_remote_eph_pages_succ_get;
+extern ssize_t ramster_remote_pers_pages_succ_get;
+extern ssize_t ramster_remote_eph_pages_unsucc_get;
+extern ssize_t ramster_remote_pers_pages_unsucc_get;
+extern ssize_t ramster_pers_pages_remote_nomem;
+extern ssize_t ramster_remote_objects_flushed;
+extern ssize_t ramster_remote_object_flushes_failed;
+extern ssize_t ramster_remote_pages_flushed;
+extern ssize_t ramster_remote_page_flushes_failed;
+
+int ramster_debugfs_init(void);
+
+static inline void inc_ramster_eph_pages_remoted(void)
+{
+ ramster_eph_pages_remoted++;
+};
+static inline void inc_ramster_pers_pages_remoted(void)
+{
+ ramster_pers_pages_remoted++;
+};
+static inline void inc_ramster_eph_pages_remote_failed(void)
+{
+ ramster_eph_pages_remote_failed++;
+};
+static inline void inc_ramster_pers_pages_remote_failed(void)
+{
+ ramster_pers_pages_remote_failed++;
+};
+static inline void inc_ramster_remote_eph_pages_succ_get(void)
+{
+ ramster_remote_eph_pages_succ_get++;
+};
+static inline void inc_ramster_remote_pers_pages_succ_get(void)
+{
+ ramster_remote_pers_pages_succ_get++;
+};
+static inline void inc_ramster_remote_eph_pages_unsucc_get(void)
+{
+ ramster_remote_eph_pages_unsucc_get++;
+};
+static inline void inc_ramster_remote_pers_pages_unsucc_get(void)
+{
+ ramster_remote_pers_pages_unsucc_get++;
+};
+static inline void inc_ramster_pers_pages_remote_nomem(void)
+{
+ ramster_pers_pages_remote_nomem++;
+};
+static inline void inc_ramster_remote_objects_flushed(void)
+{
+ ramster_remote_objects_flushed++;
+};
+static inline void inc_ramster_remote_object_flushes_failed(void)
+{
+ ramster_remote_object_flushes_failed++;
+};
+static inline void inc_ramster_remote_pages_flushed(void)
+{
+ ramster_remote_pages_flushed++;
+};
+static inline void inc_ramster_remote_page_flushes_failed(void)
+{
+ ramster_remote_page_flushes_failed++;
+};
+
+#else
+
+static inline void inc_ramster_flnodes(void) { };
+static inline void dec_ramster_flnodes(void) { };
+static inline void inc_ramster_foreign_eph_pages(void) { };
+static inline void dec_ramster_foreign_eph_pages(void) { };
+static inline void inc_ramster_foreign_pers_pages(void) { };
+static inline void dec_ramster_foreign_pers_pages(void) { };
+
+static inline void inc_ramster_eph_pages_remoted(void) { };
+static inline void inc_ramster_pers_pages_remoted(void) { };
+static inline void inc_ramster_eph_pages_remote_failed(void) { };
+static inline void inc_ramster_pers_pages_remote_failed(void) { };
+static inline void inc_ramster_remote_eph_pages_succ_get(void) { };
+static inline void inc_ramster_remote_pers_pages_succ_get(void) { };
+static inline void inc_ramster_remote_eph_pages_unsucc_get(void) { };
+static inline void inc_ramster_remote_pers_pages_unsucc_get(void) { };
+static inline void inc_ramster_pers_pages_remote_nomem(void) { };
+static inline void inc_ramster_remote_objects_flushed(void) { };
+static inline void inc_ramster_remote_object_flushes_failed(void) { };
+static inline void inc_ramster_remote_pages_flushed(void) { };
+static inline void inc_ramster_remote_page_flushes_failed(void) { };
+
+static inline int ramster_debugfs_init(void)
+{
+ return 0;
+}
+#endif
diff --git a/drivers/staging/zcache/ramster/ramster.c b/drivers/staging/zcache/ramster/ramster.c
index bf96a1cbf7c1..87816279ce3c 100644
--- a/drivers/staging/zcache/ramster/ramster.c
+++ b/drivers/staging/zcache/ramster/ramster.c
@@ -42,6 +42,7 @@
#include "ramster.h"
#include "ramster_nodemanager.h"
#include "tcp.h"
+#include "debug.h"
#define RAMSTER_TESTING
@@ -63,79 +64,12 @@ static atomic_t ramster_remote_pers_pages = ATOMIC_INIT(0);
static bool ramster_nodes_manual_up[MANUAL_NODES] __read_mostly;
static int ramster_remote_target_nodenum __read_mostly = -1;
-/* these counters are made available via debugfs */
-static long ramster_flnodes;
-static atomic_t ramster_flnodes_atomic = ATOMIC_INIT(0);
-static unsigned long ramster_flnodes_max;
-static ssize_t ramster_foreign_eph_pages;
-static atomic_t ramster_foreign_eph_pages_atomic = ATOMIC_INIT(0);
-static ssize_t ramster_foreign_eph_pages_max;
-static ssize_t ramster_foreign_pers_pages;
-static atomic_t ramster_foreign_pers_pages_atomic = ATOMIC_INIT(0);
-static ssize_t ramster_foreign_pers_pages_max;
-static ssize_t ramster_eph_pages_remoted;
-static ssize_t ramster_pers_pages_remoted;
-static ssize_t ramster_eph_pages_remote_failed;
-static ssize_t ramster_pers_pages_remote_failed;
-static ssize_t ramster_remote_eph_pages_succ_get;
-static ssize_t ramster_remote_pers_pages_succ_get;
-static ssize_t ramster_remote_eph_pages_unsucc_get;
-static ssize_t ramster_remote_pers_pages_unsucc_get;
-static ssize_t ramster_pers_pages_remote_nomem;
-static ssize_t ramster_remote_objects_flushed;
-static ssize_t ramster_remote_object_flushes_failed;
-static ssize_t ramster_remote_pages_flushed;
-static ssize_t ramster_remote_page_flushes_failed;
+/* Used by this code. */
+long ramster_flnodes;
+ssize_t ramster_foreign_eph_pages;
+ssize_t ramster_foreign_pers_pages;
/* FIXME frontswap selfshrinking knobs in debugfs? */
-#ifdef CONFIG_DEBUG_FS
-#include <linux/debugfs.h>
-#define zdfs debugfs_create_size_t
-#define zdfs64 debugfs_create_u64
-static int __init ramster_debugfs_init(void)
-{
- struct dentry *root = debugfs_create_dir("ramster", NULL);
- if (root == NULL)
- return -ENXIO;
-
- zdfs("eph_pages_remoted", S_IRUGO, root, &ramster_eph_pages_remoted);
- zdfs("pers_pages_remoted", S_IRUGO, root, &ramster_pers_pages_remoted);
- zdfs("eph_pages_remote_failed", S_IRUGO, root,
- &ramster_eph_pages_remote_failed);
- zdfs("pers_pages_remote_failed", S_IRUGO, root,
- &ramster_pers_pages_remote_failed);
- zdfs("remote_eph_pages_succ_get", S_IRUGO, root,
- &ramster_remote_eph_pages_succ_get);
- zdfs("remote_pers_pages_succ_get", S_IRUGO, root,
- &ramster_remote_pers_pages_succ_get);
- zdfs("remote_eph_pages_unsucc_get", S_IRUGO, root,
- &ramster_remote_eph_pages_unsucc_get);
- zdfs("remote_pers_pages_unsucc_get", S_IRUGO, root,
- &ramster_remote_pers_pages_unsucc_get);
- zdfs("pers_pages_remote_nomem", S_IRUGO, root,
- &ramster_pers_pages_remote_nomem);
- zdfs("remote_objects_flushed", S_IRUGO, root,
- &ramster_remote_objects_flushed);
- zdfs("remote_pages_flushed", S_IRUGO, root,
- &ramster_remote_pages_flushed);
- zdfs("remote_object_flushes_failed", S_IRUGO, root,
- &ramster_remote_object_flushes_failed);
- zdfs("remote_page_flushes_failed", S_IRUGO, root,
- &ramster_remote_page_flushes_failed);
- zdfs("foreign_eph_pages", S_IRUGO, root,
- &ramster_foreign_eph_pages);
- zdfs("foreign_eph_pages_max", S_IRUGO, root,
- &ramster_foreign_eph_pages_max);
- zdfs("foreign_pers_pages", S_IRUGO, root,
- &ramster_foreign_pers_pages);
- zdfs("foreign_pers_pages_max", S_IRUGO, root,
- &ramster_foreign_pers_pages_max);
- return 0;
-}
-#undef zdebugfs
-#undef zdfs64
-#endif
-
static LIST_HEAD(ramster_rem_op_list);
static DEFINE_SPINLOCK(ramster_rem_op_list_lock);
static DEFINE_PER_CPU(struct ramster_preload, ramster_preloads);
@@ -154,9 +88,7 @@ static struct flushlist_node *ramster_flnode_alloc(struct tmem_pool *pool)
flnode = kp->flnode;
BUG_ON(flnode == NULL);
kp->flnode = NULL;
- ramster_flnodes = atomic_inc_return(&ramster_flnodes_atomic);
- if (ramster_flnodes > ramster_flnodes_max)
- ramster_flnodes_max = ramster_flnodes;
+ inc_ramster_flnodes();
return flnode;
}
@@ -165,10 +97,8 @@ static struct flushlist_node *ramster_flnode_alloc(struct tmem_pool *pool)
static void ramster_flnode_free(struct flushlist_node *flnode,
struct tmem_pool *pool)
{
- int flnodes;
-
- flnodes = atomic_dec_return(&ramster_flnodes_atomic);
- BUG_ON(flnodes < 0);
+ dec_ramster_flnodes();
+ BUG_ON(ramster_flnodes < 0);
kmem_cache_free(ramster_flnode_cache, flnode);
}
@@ -226,9 +156,9 @@ int ramster_localify(int pool_id, struct tmem_oid *oidp, uint32_t index,
pr_err("UNTESTED pampd==NULL in ramster_localify\n");
#endif
if (eph)
- ramster_remote_eph_pages_unsucc_get++;
+ inc_ramster_remote_eph_pages_unsucc_get();
else
- ramster_remote_pers_pages_unsucc_get++;
+ inc_ramster_remote_pers_pages_unsucc_get();
obj = NULL;
goto finish;
} else if (unlikely(!pampd_is_remote(pampd))) {
@@ -237,9 +167,9 @@ int ramster_localify(int pool_id, struct tmem_oid *oidp, uint32_t index,
pr_err("UNTESTED dup while waiting in ramster_localify\n");
#endif
if (eph)
- ramster_remote_eph_pages_unsucc_get++;
+ inc_ramster_remote_eph_pages_unsucc_get();
else
- ramster_remote_pers_pages_unsucc_get++;
+ inc_ramster_remote_pers_pages_unsucc_get();
obj = NULL;
pampd = NULL;
ret = -EEXIST;
@@ -248,7 +178,7 @@ int ramster_localify(int pool_id, struct tmem_oid *oidp, uint32_t index,
/* no remote data, delete the local is_remote pampd */
pampd = NULL;
if (eph)
- ramster_remote_eph_pages_unsucc_get++;
+ inc_ramster_remote_eph_pages_unsucc_get();
else
BUG();
delete = true;
@@ -279,9 +209,9 @@ int ramster_localify(int pool_id, struct tmem_oid *oidp, uint32_t index,
BUG_ON(extra == NULL);
zcache_decompress_to_page(data, size, (struct page *)extra);
if (eph)
- ramster_remote_eph_pages_succ_get++;
+ inc_ramster_remote_eph_pages_succ_get();
else
- ramster_remote_pers_pages_succ_get++;
+ inc_ramster_remote_pers_pages_succ_get();
ret = 0;
finish:
tmem_localify_finish(obj, index, pampd, saved_hb, delete);
@@ -366,7 +296,7 @@ void *ramster_pampd_repatriate_preload(void *pampd, struct tmem_pool *pool,
c = atomic_dec_return(&ramster_remote_pers_pages);
WARN_ON_ONCE(c < 0);
} else {
- ramster_pers_pages_remote_nomem++;
+ inc_ramster_pers_pages_remote_nomem();
}
local_irq_restore(flags);
out:
@@ -461,32 +391,21 @@ void *ramster_pampd_free(void *pampd, struct tmem_pool *pool,
void ramster_count_foreign_pages(bool eph, int count)
{
- int c;
-
BUG_ON(count != 1 && count != -1);
if (eph) {
if (count > 0) {
- c = atomic_inc_return(
- &ramster_foreign_eph_pages_atomic);
- if (c > ramster_foreign_eph_pages_max)
- ramster_foreign_eph_pages_max = c;
+ inc_ramster_foreign_eph_pages();
} else {
- c = atomic_dec_return(&ramster_foreign_eph_pages_atomic);
- WARN_ON_ONCE(c < 0);
+ dec_ramster_foreign_eph_pages();
+ WARN_ON_ONCE(ramster_foreign_eph_pages < 0);
}
- ramster_foreign_eph_pages = c;
} else {
if (count > 0) {
- c = atomic_inc_return(
- &ramster_foreign_pers_pages_atomic);
- if (c > ramster_foreign_pers_pages_max)
- ramster_foreign_pers_pages_max = c;
+ inc_ramster_foreign_pers_pages();
} else {
- c = atomic_dec_return(
- &ramster_foreign_pers_pages_atomic);
- WARN_ON_ONCE(c < 0);
+ dec_ramster_foreign_pers_pages();
+ WARN_ON_ONCE(ramster_foreign_pers_pages < 0);
}
- ramster_foreign_pers_pages = c;
}
}
@@ -516,9 +435,9 @@ static void ramster_remote_flush_page(struct flushlist_node *flnode)
remotenode = flnode->xh.client_id;
ret = r2net_remote_flush(xh, remotenode);
if (ret >= 0)
- ramster_remote_pages_flushed++;
+ inc_ramster_remote_pages_flushed();
else
- ramster_remote_page_flushes_failed++;
+ inc_ramster_remote_page_flushes_failed();
preempt_enable_no_resched();
ramster_flnode_free(flnode, NULL);
}
@@ -533,9 +452,9 @@ static void ramster_remote_flush_object(struct flushlist_node *flnode)
remotenode = flnode->xh.client_id;
ret = r2net_remote_flush_object(xh, remotenode);
if (ret >= 0)
- ramster_remote_objects_flushed++;
+ inc_ramster_remote_objects_flushed();
else
- ramster_remote_object_flushes_failed++;
+ inc_ramster_remote_object_flushes_failed();
preempt_enable_no_resched();
ramster_flnode_free(flnode, NULL);
}
@@ -586,18 +505,18 @@ int ramster_remotify_pageframe(bool eph)
* But count them so we know if it becomes a problem.
*/
if (eph)
- ramster_eph_pages_remote_failed++;
+ inc_ramster_eph_pages_remote_failed();
else
- ramster_pers_pages_remote_failed++;
+ inc_ramster_pers_pages_remote_failed();
break;
} else {
if (!eph)
atomic_inc(&ramster_remote_pers_pages);
}
if (eph)
- ramster_eph_pages_remoted++;
+ inc_ramster_eph_pages_remoted();
else
- ramster_pers_pages_remoted++;
+ inc_ramster_pers_pages_remoted();
/*
* data was successfully remoted so change the local version to
* point to the remote node where it landed
diff --git a/drivers/staging/zcache/ramster/tcp.c b/drivers/staging/zcache/ramster/tcp.c
index aa2a1a763aa4..f6e1e5209d88 100644
--- a/drivers/staging/zcache/ramster/tcp.c
+++ b/drivers/staging/zcache/ramster/tcp.c
@@ -300,27 +300,22 @@ static u8 r2net_num_from_nn(struct r2net_node *nn)
static int r2net_prep_nsw(struct r2net_node *nn, struct r2net_status_wait *nsw)
{
- int ret = 0;
+ int ret;
- do {
- if (!idr_pre_get(&nn->nn_status_idr, GFP_ATOMIC)) {
- ret = -EAGAIN;
- break;
- }
- spin_lock(&nn->nn_lock);
- ret = idr_get_new(&nn->nn_status_idr, nsw, &nsw->ns_id);
- if (ret == 0)
- list_add_tail(&nsw->ns_node_item,
- &nn->nn_status_list);
- spin_unlock(&nn->nn_lock);
- } while (ret == -EAGAIN);
+ spin_lock(&nn->nn_lock);
+ ret = idr_alloc(&nn->nn_status_idr, nsw, 0, 0, GFP_ATOMIC);
+ if (ret >= 0) {
+ nsw->ns_id = ret;
+ list_add_tail(&nsw->ns_node_item, &nn->nn_status_list);
+ }
+ spin_unlock(&nn->nn_lock);
- if (ret == 0) {
+ if (ret >= 0) {
init_waitqueue_head(&nsw->ns_wq);
nsw->ns_sys_status = R2NET_ERR_NONE;
nsw->ns_status = 0;
+ return 0;
}
-
return ret;
}
diff --git a/drivers/staging/zcache/zbud.c b/drivers/staging/zcache/zbud.c
index fdff5c6a0239..6cda4ed9ed31 100644
--- a/drivers/staging/zcache/zbud.c
+++ b/drivers/staging/zcache/zbud.c
@@ -342,6 +342,11 @@ static int zbud_debugfs_init(void)
}
#undef zdfs
#undef zdfs64
+#else
+static inline int zbud_debugfs_init(void)
+{
+ return 0;
+}
#endif
/* protects the buddied list and all unbuddied lists */
@@ -1051,9 +1056,7 @@ void zbud_init(void)
{
int i;
-#ifdef CONFIG_DEBUG_FS
zbud_debugfs_init();
-#endif
BUG_ON((sizeof(struct tmem_handle) * 2 > CHUNK_SIZE));
BUG_ON(sizeof(struct zbudpage) > sizeof(struct page));
for (i = 0; i < NCHUNKS; i++) {
diff --git a/drivers/staging/zcache/zcache-main.c b/drivers/staging/zcache/zcache-main.c
index 328898ea76c3..e23d814b5392 100644
--- a/drivers/staging/zcache/zcache-main.c
+++ b/drivers/staging/zcache/zcache-main.c
@@ -19,6 +19,7 @@
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/types.h>
+#include <linux/string.h>
#include <linux/atomic.h>
#include <linux/math64.h>
#include <linux/crypto.h>
@@ -33,10 +34,11 @@
#include "zcache.h"
#include "zbud.h"
#include "ramster.h"
+#include "debug.h"
#ifdef CONFIG_RAMSTER
-static int ramster_enabled;
+static bool ramster_enabled __read_mostly;
#else
-#define ramster_enabled 0
+#define ramster_enabled false
#endif
#ifndef __PG_WAS_ACTIVE
@@ -59,21 +61,25 @@ static inline void frontswap_tmem_exclusive_gets(bool b)
}
#endif
+/*
+ * mark pampd to special value in order that later
+ * retrieve will identify zero-filled pages
+ */
+#define ZERO_FILLED 0x2
+
/* enable (or fix code) when Seth's patches are accepted upstream */
#define zcache_writeback_enabled 0
-static int zcache_enabled __read_mostly;
-static int disable_cleancache __read_mostly;
-static int disable_frontswap __read_mostly;
-static int disable_frontswap_ignore_nonactive __read_mostly;
-static int disable_cleancache_ignore_nonactive __read_mostly;
+static bool zcache_enabled __read_mostly;
+static bool disable_cleancache __read_mostly;
+static bool disable_frontswap __read_mostly;
+static bool disable_frontswap_ignore_nonactive __read_mostly;
+static bool disable_cleancache_ignore_nonactive __read_mostly;
static char *namestr __read_mostly = "zcache";
#define ZCACHE_GFP_MASK \
(__GFP_FS | __GFP_NORETRY | __GFP_NOWARN | __GFP_NOMEMALLOC)
-MODULE_LICENSE("GPL");
-
/* crypto API for zcache */
#define ZCACHE_COMP_NAME_SZ CRYPTO_MAX_ALG_NAME
static char zcache_comp_name[ZCACHE_COMP_NAME_SZ] __read_mostly;
@@ -134,196 +140,21 @@ static struct kmem_cache *zcache_obj_cache;
static DEFINE_PER_CPU(struct zcache_preload, zcache_preloads) = { 0, };
-/* we try to keep these statistics SMP-consistent */
-static ssize_t zcache_obj_count;
-static atomic_t zcache_obj_atomic = ATOMIC_INIT(0);
-static ssize_t zcache_obj_count_max;
-static ssize_t zcache_objnode_count;
-static atomic_t zcache_objnode_atomic = ATOMIC_INIT(0);
-static ssize_t zcache_objnode_count_max;
-static u64 zcache_eph_zbytes;
-static atomic_long_t zcache_eph_zbytes_atomic = ATOMIC_INIT(0);
-static u64 zcache_eph_zbytes_max;
-static u64 zcache_pers_zbytes;
-static atomic_long_t zcache_pers_zbytes_atomic = ATOMIC_INIT(0);
-static u64 zcache_pers_zbytes_max;
-static ssize_t zcache_eph_pageframes;
-static atomic_t zcache_eph_pageframes_atomic = ATOMIC_INIT(0);
-static ssize_t zcache_eph_pageframes_max;
-static ssize_t zcache_pers_pageframes;
-static atomic_t zcache_pers_pageframes_atomic = ATOMIC_INIT(0);
-static ssize_t zcache_pers_pageframes_max;
-static ssize_t zcache_pageframes_alloced;
-static atomic_t zcache_pageframes_alloced_atomic = ATOMIC_INIT(0);
-static ssize_t zcache_pageframes_freed;
-static atomic_t zcache_pageframes_freed_atomic = ATOMIC_INIT(0);
-static ssize_t zcache_eph_zpages;
-static ssize_t zcache_eph_zpages;
-static atomic_t zcache_eph_zpages_atomic = ATOMIC_INIT(0);
-static ssize_t zcache_eph_zpages_max;
-static ssize_t zcache_pers_zpages;
-static atomic_t zcache_pers_zpages_atomic = ATOMIC_INIT(0);
-static ssize_t zcache_pers_zpages_max;
-
-/* but for the rest of these, counting races are ok */
-static ssize_t zcache_flush_total;
-static ssize_t zcache_flush_found;
-static ssize_t zcache_flobj_total;
-static ssize_t zcache_flobj_found;
-static ssize_t zcache_failed_eph_puts;
-static ssize_t zcache_failed_pers_puts;
-static ssize_t zcache_failed_getfreepages;
-static ssize_t zcache_failed_alloc;
-static ssize_t zcache_put_to_flush;
-static ssize_t zcache_compress_poor;
-static ssize_t zcache_mean_compress_poor;
-static ssize_t zcache_eph_ate_tail;
-static ssize_t zcache_eph_ate_tail_failed;
-static ssize_t zcache_pers_ate_eph;
-static ssize_t zcache_pers_ate_eph_failed;
-static ssize_t zcache_evicted_eph_zpages;
-static ssize_t zcache_evicted_eph_pageframes;
-static ssize_t zcache_last_active_file_pageframes;
-static ssize_t zcache_last_inactive_file_pageframes;
-static ssize_t zcache_last_active_anon_pageframes;
-static ssize_t zcache_last_inactive_anon_pageframes;
-static ssize_t zcache_eph_nonactive_puts_ignored;
-static ssize_t zcache_pers_nonactive_puts_ignored;
-static ssize_t zcache_writtenback_pages;
-static ssize_t zcache_outstanding_writeback_pages;
-
-#ifdef CONFIG_DEBUG_FS
-#include <linux/debugfs.h>
-#define zdfs debugfs_create_size_t
-#define zdfs64 debugfs_create_u64
-static int zcache_debugfs_init(void)
-{
- struct dentry *root = debugfs_create_dir("zcache", NULL);
- if (root == NULL)
- return -ENXIO;
-
- zdfs("obj_count", S_IRUGO, root, &zcache_obj_count);
- zdfs("obj_count_max", S_IRUGO, root, &zcache_obj_count_max);
- zdfs("objnode_count", S_IRUGO, root, &zcache_objnode_count);
- zdfs("objnode_count_max", S_IRUGO, root, &zcache_objnode_count_max);
- zdfs("flush_total", S_IRUGO, root, &zcache_flush_total);
- zdfs("flush_found", S_IRUGO, root, &zcache_flush_found);
- zdfs("flobj_total", S_IRUGO, root, &zcache_flobj_total);
- zdfs("flobj_found", S_IRUGO, root, &zcache_flobj_found);
- zdfs("failed_eph_puts", S_IRUGO, root, &zcache_failed_eph_puts);
- zdfs("failed_pers_puts", S_IRUGO, root, &zcache_failed_pers_puts);
- zdfs("failed_get_free_pages", S_IRUGO, root,
- &zcache_failed_getfreepages);
- zdfs("failed_alloc", S_IRUGO, root, &zcache_failed_alloc);
- zdfs("put_to_flush", S_IRUGO, root, &zcache_put_to_flush);
- zdfs("compress_poor", S_IRUGO, root, &zcache_compress_poor);
- zdfs("mean_compress_poor", S_IRUGO, root, &zcache_mean_compress_poor);
- zdfs("eph_ate_tail", S_IRUGO, root, &zcache_eph_ate_tail);
- zdfs("eph_ate_tail_failed", S_IRUGO, root, &zcache_eph_ate_tail_failed);
- zdfs("pers_ate_eph", S_IRUGO, root, &zcache_pers_ate_eph);
- zdfs("pers_ate_eph_failed", S_IRUGO, root, &zcache_pers_ate_eph_failed);
- zdfs("evicted_eph_zpages", S_IRUGO, root, &zcache_evicted_eph_zpages);
- zdfs("evicted_eph_pageframes", S_IRUGO, root,
- &zcache_evicted_eph_pageframes);
- zdfs("eph_pageframes", S_IRUGO, root, &zcache_eph_pageframes);
- zdfs("eph_pageframes_max", S_IRUGO, root, &zcache_eph_pageframes_max);
- zdfs("pers_pageframes", S_IRUGO, root, &zcache_pers_pageframes);
- zdfs("pers_pageframes_max", S_IRUGO, root, &zcache_pers_pageframes_max);
- zdfs("eph_zpages", S_IRUGO, root, &zcache_eph_zpages);
- zdfs("eph_zpages_max", S_IRUGO, root, &zcache_eph_zpages_max);
- zdfs("pers_zpages", S_IRUGO, root, &zcache_pers_zpages);
- zdfs("pers_zpages_max", S_IRUGO, root, &zcache_pers_zpages_max);
- zdfs("last_active_file_pageframes", S_IRUGO, root,
- &zcache_last_active_file_pageframes);
- zdfs("last_inactive_file_pageframes", S_IRUGO, root,
- &zcache_last_inactive_file_pageframes);
- zdfs("last_active_anon_pageframes", S_IRUGO, root,
- &zcache_last_active_anon_pageframes);
- zdfs("last_inactive_anon_pageframes", S_IRUGO, root,
- &zcache_last_inactive_anon_pageframes);
- zdfs("eph_nonactive_puts_ignored", S_IRUGO, root,
- &zcache_eph_nonactive_puts_ignored);
- zdfs("pers_nonactive_puts_ignored", S_IRUGO, root,
- &zcache_pers_nonactive_puts_ignored);
- zdfs64("eph_zbytes", S_IRUGO, root, &zcache_eph_zbytes);
- zdfs64("eph_zbytes_max", S_IRUGO, root, &zcache_eph_zbytes_max);
- zdfs64("pers_zbytes", S_IRUGO, root, &zcache_pers_zbytes);
- zdfs64("pers_zbytes_max", S_IRUGO, root, &zcache_pers_zbytes_max);
- zdfs("outstanding_writeback_pages", S_IRUGO, root,
- &zcache_outstanding_writeback_pages);
- zdfs("writtenback_pages", S_IRUGO, root, &zcache_writtenback_pages);
- return 0;
-}
-#undef zdebugfs
-#undef zdfs64
-#endif
-
-#define ZCACHE_DEBUG
-#ifdef ZCACHE_DEBUG
-/* developers can call this in case of ooms, e.g. to find memory leaks */
-void zcache_dump(void)
-{
- pr_info("zcache: obj_count=%zd\n", zcache_obj_count);
- pr_info("zcache: obj_count_max=%zd\n", zcache_obj_count_max);
- pr_info("zcache: objnode_count=%zd\n", zcache_objnode_count);
- pr_info("zcache: objnode_count_max=%zd\n", zcache_objnode_count_max);
- pr_info("zcache: flush_total=%zd\n", zcache_flush_total);
- pr_info("zcache: flush_found=%zd\n", zcache_flush_found);
- pr_info("zcache: flobj_total=%zd\n", zcache_flobj_total);
- pr_info("zcache: flobj_found=%zd\n", zcache_flobj_found);
- pr_info("zcache: failed_eph_puts=%zd\n", zcache_failed_eph_puts);
- pr_info("zcache: failed_pers_puts=%zd\n", zcache_failed_pers_puts);
- pr_info("zcache: failed_get_free_pages=%zd\n",
- zcache_failed_getfreepages);
- pr_info("zcache: failed_alloc=%zd\n", zcache_failed_alloc);
- pr_info("zcache: put_to_flush=%zd\n", zcache_put_to_flush);
- pr_info("zcache: compress_poor=%zd\n", zcache_compress_poor);
- pr_info("zcache: mean_compress_poor=%zd\n",
- zcache_mean_compress_poor);
- pr_info("zcache: eph_ate_tail=%zd\n", zcache_eph_ate_tail);
- pr_info("zcache: eph_ate_tail_failed=%zd\n",
- zcache_eph_ate_tail_failed);
- pr_info("zcache: pers_ate_eph=%zd\n", zcache_pers_ate_eph);
- pr_info("zcache: pers_ate_eph_failed=%zd\n",
- zcache_pers_ate_eph_failed);
- pr_info("zcache: evicted_eph_zpages=%zd\n", zcache_evicted_eph_zpages);
- pr_info("zcache: evicted_eph_pageframes=%zd\n",
- zcache_evicted_eph_pageframes);
- pr_info("zcache: eph_pageframes=%zd\n", zcache_eph_pageframes);
- pr_info("zcache: eph_pageframes_max=%zd\n", zcache_eph_pageframes_max);
- pr_info("zcache: pers_pageframes=%zd\n", zcache_pers_pageframes);
- pr_info("zcache: pers_pageframes_max=%zd\n",
- zcache_pers_pageframes_max);
- pr_info("zcache: eph_zpages=%zd\n", zcache_eph_zpages);
- pr_info("zcache: eph_zpages_max=%zd\n", zcache_eph_zpages_max);
- pr_info("zcache: pers_zpages=%zd\n", zcache_pers_zpages);
- pr_info("zcache: pers_zpages_max=%zd\n", zcache_pers_zpages_max);
- pr_info("zcache: last_active_file_pageframes=%zd\n",
- zcache_last_active_file_pageframes);
- pr_info("zcache: last_inactive_file_pageframes=%zd\n",
- zcache_last_inactive_file_pageframes);
- pr_info("zcache: last_active_anon_pageframes=%zd\n",
- zcache_last_active_anon_pageframes);
- pr_info("zcache: last_inactive_anon_pageframes=%zd\n",
- zcache_last_inactive_anon_pageframes);
- pr_info("zcache: eph_nonactive_puts_ignored=%zd\n",
- zcache_eph_nonactive_puts_ignored);
- pr_info("zcache: pers_nonactive_puts_ignored=%zd\n",
- zcache_pers_nonactive_puts_ignored);
- pr_info("zcache: eph_zbytes=%llu\n",
- zcache_eph_zbytes);
- pr_info("zcache: eph_zbytes_max=%llu\n",
- zcache_eph_zbytes_max);
- pr_info("zcache: pers_zbytes=%llu\n",
- zcache_pers_zbytes);
- pr_info("zcache: pers_zbytes_max=%llu\n",
- zcache_pers_zbytes_max);
- pr_info("zcache: outstanding_writeback_pages=%zd\n",
- zcache_outstanding_writeback_pages);
- pr_info("zcache: writtenback_pages=%zd\n", zcache_writtenback_pages);
-}
+/* Used by debug.c */
+ssize_t zcache_pers_zpages;
+u64 zcache_pers_zbytes;
+ssize_t zcache_eph_pageframes;
+ssize_t zcache_pers_pageframes;
+
+/* Used by this code. */
+ssize_t zcache_last_active_file_pageframes;
+ssize_t zcache_last_inactive_file_pageframes;
+ssize_t zcache_last_active_anon_pageframes;
+ssize_t zcache_last_inactive_anon_pageframes;
+#ifdef CONFIG_ZCACHE_WRITEBACK
+ssize_t zcache_writtenback_pages;
+ssize_t zcache_outstanding_writeback_pages;
#endif
-
/*
* zcache core code starts here
*/
@@ -422,18 +253,14 @@ static struct tmem_objnode *zcache_objnode_alloc(struct tmem_pool *pool)
}
}
BUG_ON(objnode == NULL);
- zcache_objnode_count = atomic_inc_return(&zcache_objnode_atomic);
- if (zcache_objnode_count > zcache_objnode_count_max)
- zcache_objnode_count_max = zcache_objnode_count;
+ inc_zcache_objnode_count();
return objnode;
}
static void zcache_objnode_free(struct tmem_objnode *objnode,
struct tmem_pool *pool)
{
- zcache_objnode_count =
- atomic_dec_return(&zcache_objnode_atomic);
- BUG_ON(zcache_objnode_count < 0);
+ dec_zcache_objnode_count();
kmem_cache_free(zcache_objnode_cache, objnode);
}
@@ -446,20 +273,49 @@ static struct tmem_obj *zcache_obj_alloc(struct tmem_pool *pool)
obj = kp->obj;
BUG_ON(obj == NULL);
kp->obj = NULL;
- zcache_obj_count = atomic_inc_return(&zcache_obj_atomic);
- if (zcache_obj_count > zcache_obj_count_max)
- zcache_obj_count_max = zcache_obj_count;
+ inc_zcache_obj_count();
return obj;
}
static void zcache_obj_free(struct tmem_obj *obj, struct tmem_pool *pool)
{
- zcache_obj_count =
- atomic_dec_return(&zcache_obj_atomic);
- BUG_ON(zcache_obj_count < 0);
+ dec_zcache_obj_count();
kmem_cache_free(zcache_obj_cache, obj);
}
+/*
+ * Compressing zero-filled pages will waste memory and introduce
+ * serious fragmentation, skip it to avoid overhead.
+ */
+static bool page_is_zero_filled(struct page *p)
+{
+ unsigned int pos;
+ char *page;
+
+ page = kmap_atomic(p);
+ for (pos = 0; pos < PAGE_SIZE / sizeof(*page); pos++) {
+ if (page[pos]) {
+ kunmap_atomic(page);
+ return false;
+ }
+ }
+ kunmap_atomic(page);
+
+ return true;
+}
+
+static void handle_zero_filled_page(void *p)
+{
+ void *user_mem;
+ struct page *page = (struct page *)p;
+
+ user_mem = kmap_atomic(page);
+ memset(user_mem, 0, PAGE_SIZE);
+ kunmap_atomic(user_mem);
+
+ flush_dcache_page(page);
+}
+
static struct tmem_hostops zcache_hostops = {
.obj_alloc = zcache_obj_alloc,
.obj_free = zcache_obj_free,
@@ -472,8 +328,7 @@ static struct page *zcache_alloc_page(void)
struct page *page = alloc_page(ZCACHE_GFP_MASK);
if (page != NULL)
- zcache_pageframes_alloced =
- atomic_inc_return(&zcache_pageframes_alloced_atomic);
+ inc_zcache_pageframes_alloced();
return page;
}
@@ -485,17 +340,13 @@ static void zcache_free_page(struct page *page)
if (page == NULL)
BUG();
__free_page(page);
- zcache_pageframes_freed =
- atomic_inc_return(&zcache_pageframes_freed_atomic);
- curr_pageframes = zcache_pageframes_alloced -
- atomic_read(&zcache_pageframes_freed_atomic) -
- atomic_read(&zcache_eph_pageframes_atomic) -
- atomic_read(&zcache_pers_pageframes_atomic);
+ inc_zcache_pageframes_freed();
+ curr_pageframes = curr_pageframes_count();
if (curr_pageframes > max_pageframes)
max_pageframes = curr_pageframes;
if (curr_pageframes < min_pageframes)
min_pageframes = curr_pageframes;
-#ifdef ZCACHE_DEBUG
+#ifdef CONFIG_ZCACHE_DEBUG
if (curr_pageframes > 2L || curr_pageframes < -2L) {
/* pr_info here */
}
@@ -517,12 +368,20 @@ static void *zcache_pampd_eph_create(char *data, size_t size, bool raw,
{
void *pampd = NULL, *cdata = data;
unsigned clen = size;
+ bool zero_filled = false;
struct page *page = (struct page *)(data), *newpage;
+ if (page_is_zero_filled(page)) {
+ clen = 0;
+ zero_filled = true;
+ inc_zcache_zero_filled_pages();
+ goto got_pampd;
+ }
+
if (!raw) {
zcache_compress(page, &cdata, &clen);
if (clen > zbud_max_buddy_size()) {
- zcache_compress_poor++;
+ inc_zcache_compress_poor();
goto out;
}
} else {
@@ -539,33 +398,27 @@ static void *zcache_pampd_eph_create(char *data, size_t size, bool raw,
if (newpage != NULL)
goto create_in_new_page;
- zcache_failed_getfreepages++;
+ inc_zcache_failed_getfreepages();
/* can't allocate a page, evict an ephemeral page via LRU */
newpage = zcache_evict_eph_pageframe();
if (newpage == NULL) {
- zcache_eph_ate_tail_failed++;
+ inc_zcache_eph_ate_tail_failed();
goto out;
}
- zcache_eph_ate_tail++;
+ inc_zcache_eph_ate_tail();
create_in_new_page:
pampd = (void *)zbud_create_prep(th, true, cdata, clen, newpage);
BUG_ON(pampd == NULL);
- zcache_eph_pageframes =
- atomic_inc_return(&zcache_eph_pageframes_atomic);
- if (zcache_eph_pageframes > zcache_eph_pageframes_max)
- zcache_eph_pageframes_max = zcache_eph_pageframes;
+ inc_zcache_eph_pageframes();
got_pampd:
- zcache_eph_zbytes =
- atomic_long_add_return(clen, &zcache_eph_zbytes_atomic);
- if (zcache_eph_zbytes > zcache_eph_zbytes_max)
- zcache_eph_zbytes_max = zcache_eph_zbytes;
- zcache_eph_zpages = atomic_inc_return(&zcache_eph_zpages_atomic);
- if (zcache_eph_zpages > zcache_eph_zpages_max)
- zcache_eph_zpages_max = zcache_eph_zpages;
- if (ramster_enabled && raw)
+ inc_zcache_eph_zbytes(clen);
+ inc_zcache_eph_zpages();
+ if (ramster_enabled && raw && !zero_filled)
ramster_count_foreign_pages(true, 1);
+ if (zero_filled)
+ pampd = (void *)ZERO_FILLED;
out:
return pampd;
}
@@ -575,6 +428,7 @@ static void *zcache_pampd_pers_create(char *data, size_t size, bool raw,
{
void *pampd = NULL, *cdata = data;
unsigned clen = size;
+ bool zero_filled = false;
struct page *page = (struct page *)(data), *newpage;
unsigned long zbud_mean_zsize;
unsigned long curr_pers_zpages, total_zsize;
@@ -583,13 +437,21 @@ static void *zcache_pampd_pers_create(char *data, size_t size, bool raw,
BUG_ON(!ramster_enabled);
goto create_pampd;
}
+
+ if (page_is_zero_filled(page)) {
+ clen = 0;
+ zero_filled = true;
+ inc_zcache_zero_filled_pages();
+ goto got_pampd;
+ }
+
curr_pers_zpages = zcache_pers_zpages;
/* FIXME CONFIG_RAMSTER... subtract atomic remote_pers_pages here? */
if (!raw)
zcache_compress(page, &cdata, &clen);
/* reject if compression is too poor */
if (clen > zbud_max_zsize) {
- zcache_compress_poor++;
+ inc_zcache_compress_poor();
goto out;
}
/* reject if mean compression is too poor */
@@ -600,7 +462,7 @@ static void *zcache_pampd_pers_create(char *data, size_t size, bool raw,
zbud_mean_zsize = div_u64(total_zsize,
curr_pers_zpages);
if (zbud_mean_zsize > zbud_max_mean_zsize) {
- zcache_mean_compress_poor++;
+ inc_zcache_mean_compress_poor();
goto out;
}
}
@@ -621,33 +483,27 @@ create_pampd:
* (global_page_state(NR_LRU_BASE + LRU_ACTIVE_FILE) +
* global_page_state(NR_LRU_BASE + LRU_INACTIVE_FILE)))
*/
- zcache_failed_getfreepages++;
+ inc_zcache_failed_getfreepages();
/* can't allocate a page, evict an ephemeral page via LRU */
newpage = zcache_evict_eph_pageframe();
if (newpage == NULL) {
- zcache_pers_ate_eph_failed++;
+ inc_zcache_pers_ate_eph_failed();
goto out;
}
- zcache_pers_ate_eph++;
+ inc_zcache_pers_ate_eph();
create_in_new_page:
pampd = (void *)zbud_create_prep(th, false, cdata, clen, newpage);
BUG_ON(pampd == NULL);
- zcache_pers_pageframes =
- atomic_inc_return(&zcache_pers_pageframes_atomic);
- if (zcache_pers_pageframes > zcache_pers_pageframes_max)
- zcache_pers_pageframes_max = zcache_pers_pageframes;
+ inc_zcache_pers_pageframes();
got_pampd:
- zcache_pers_zpages = atomic_inc_return(&zcache_pers_zpages_atomic);
- if (zcache_pers_zpages > zcache_pers_zpages_max)
- zcache_pers_zpages_max = zcache_pers_zpages;
- zcache_pers_zbytes =
- atomic_long_add_return(clen, &zcache_pers_zbytes_atomic);
- if (zcache_pers_zbytes > zcache_pers_zbytes_max)
- zcache_pers_zbytes_max = zcache_pers_zbytes;
- if (ramster_enabled && raw)
+ inc_zcache_pers_zpages();
+ inc_zcache_pers_zbytes(clen);
+ if (ramster_enabled && raw && !zero_filled)
ramster_count_foreign_pages(false, 1);
+ if (zero_filled)
+ pampd = (void *)ZERO_FILLED;
out:
return pampd;
}
@@ -676,7 +532,7 @@ void *zcache_pampd_create(char *data, unsigned int size, bool raw,
objnode = kmem_cache_alloc(zcache_objnode_cache,
ZCACHE_GFP_MASK);
if (unlikely(objnode == NULL)) {
- zcache_failed_alloc++;
+ inc_zcache_failed_alloc();
goto out;
}
kp->objnodes[i] = objnode;
@@ -687,7 +543,7 @@ void *zcache_pampd_create(char *data, unsigned int size, bool raw,
kp->obj = obj;
}
if (unlikely(kp->obj == NULL)) {
- zcache_failed_alloc++;
+ inc_zcache_failed_alloc();
goto out;
}
/*
@@ -709,7 +565,8 @@ out:
*/
void zcache_pampd_create_finish(void *pampd, bool eph)
{
- zbud_create_finish((struct zbudref *)pampd, eph);
+ if (pampd != (void *)ZERO_FILLED)
+ zbud_create_finish((struct zbudref *)pampd, eph);
}
/*
@@ -754,6 +611,14 @@ static int zcache_pampd_get_data(char *data, size_t *sizep, bool raw,
BUG_ON(preemptible());
BUG_ON(eph); /* fix later if shared pools get implemented */
BUG_ON(pampd_is_remote(pampd));
+
+ if (pampd == (void *)ZERO_FILLED) {
+ handle_zero_filled_page(data);
+ if (!raw)
+ *sizep = PAGE_SIZE;
+ return 0;
+ }
+
if (raw)
ret = zbud_copy_from_zbud(data, (struct zbudref *)pampd,
sizep, eph);
@@ -774,13 +639,25 @@ static int zcache_pampd_get_data_and_free(char *data, size_t *sizep, bool raw,
void *pampd, struct tmem_pool *pool,
struct tmem_oid *oid, uint32_t index)
{
- int ret;
- bool eph = !is_persistent(pool);
+ int ret = 0;
+ bool eph = !is_persistent(pool), zero_filled = false;
struct page *page = NULL;
unsigned int zsize, zpages;
BUG_ON(preemptible());
BUG_ON(pampd_is_remote(pampd));
+
+ if (pampd == (void *)ZERO_FILLED) {
+ handle_zero_filled_page(data);
+ zero_filled = true;
+ zsize = 0;
+ zpages = 1;
+ if (!raw)
+ *sizep = PAGE_SIZE;
+ dec_zcache_zero_filled_pages();
+ goto zero_fill;
+ }
+
if (raw)
ret = zbud_copy_from_zbud(data, (struct zbudref *)pampd,
sizep, eph);
@@ -792,26 +669,21 @@ static int zcache_pampd_get_data_and_free(char *data, size_t *sizep, bool raw,
}
page = zbud_free_and_delist((struct zbudref *)pampd, eph,
&zsize, &zpages);
+zero_fill:
if (eph) {
if (page)
- zcache_eph_pageframes =
- atomic_dec_return(&zcache_eph_pageframes_atomic);
- zcache_eph_zpages =
- atomic_sub_return(zpages, &zcache_eph_zpages_atomic);
- zcache_eph_zbytes =
- atomic_long_sub_return(zsize, &zcache_eph_zbytes_atomic);
+ dec_zcache_eph_pageframes();
+ dec_zcache_eph_zpages(zpages);
+ dec_zcache_eph_zbytes(zsize);
} else {
if (page)
- zcache_pers_pageframes =
- atomic_dec_return(&zcache_pers_pageframes_atomic);
- zcache_pers_zpages =
- atomic_sub_return(zpages, &zcache_pers_zpages_atomic);
- zcache_pers_zbytes =
- atomic_long_sub_return(zsize, &zcache_pers_zbytes_atomic);
+ dec_zcache_pers_pageframes();
+ dec_zcache_pers_zpages(zpages);
+ dec_zcache_pers_zbytes(zsize);
}
- if (!is_local_client(pool->client))
+ if (!is_local_client(pool->client) && !zero_filled)
ramster_count_foreign_pages(eph, -1);
- if (page)
+ if (page && !zero_filled)
zcache_free_page(page);
return ret;
}
@@ -825,39 +697,44 @@ static void zcache_pampd_free(void *pampd, struct tmem_pool *pool,
{
struct page *page = NULL;
unsigned int zsize, zpages;
+ bool zero_filled = false;
BUG_ON(preemptible());
- if (pampd_is_remote(pampd)) {
+
+ if (pampd == (void *)ZERO_FILLED) {
+ zero_filled = true;
+ zsize = 0;
+ zpages = 1;
+ dec_zcache_zero_filled_pages();
+ }
+
+ if (pampd_is_remote(pampd) && !zero_filled) {
BUG_ON(!ramster_enabled);
pampd = ramster_pampd_free(pampd, pool, oid, index, acct);
if (pampd == NULL)
return;
}
if (is_ephemeral(pool)) {
- page = zbud_free_and_delist((struct zbudref *)pampd,
+ if (!zero_filled)
+ page = zbud_free_and_delist((struct zbudref *)pampd,
true, &zsize, &zpages);
if (page)
- zcache_eph_pageframes =
- atomic_dec_return(&zcache_eph_pageframes_atomic);
- zcache_eph_zpages =
- atomic_sub_return(zpages, &zcache_eph_zpages_atomic);
- zcache_eph_zbytes =
- atomic_long_sub_return(zsize, &zcache_eph_zbytes_atomic);
+ dec_zcache_eph_pageframes();
+ dec_zcache_eph_zpages(zpages);
+ dec_zcache_eph_zbytes(zsize);
/* FIXME CONFIG_RAMSTER... check acct parameter? */
} else {
- page = zbud_free_and_delist((struct zbudref *)pampd,
+ if (!zero_filled)
+ page = zbud_free_and_delist((struct zbudref *)pampd,
false, &zsize, &zpages);
if (page)
- zcache_pers_pageframes =
- atomic_dec_return(&zcache_pers_pageframes_atomic);
- zcache_pers_zpages =
- atomic_sub_return(zpages, &zcache_pers_zpages_atomic);
- zcache_pers_zbytes =
- atomic_long_sub_return(zsize, &zcache_pers_zbytes_atomic);
+ dec_zcache_pers_pageframes();
+ dec_zcache_pers_zpages(zpages);
+ dec_zcache_pers_zbytes(zsize);
}
- if (!is_local_client(pool->client))
+ if (!is_local_client(pool->client) && !zero_filled)
ramster_count_foreign_pages(is_ephemeral(pool), -1);
- if (page)
+ if (page && !zero_filled)
zcache_free_page(page);
}
@@ -975,14 +852,11 @@ static struct page *zcache_evict_eph_pageframe(void)
page = zbud_evict_pageframe_lru(&zsize, &zpages);
if (page == NULL)
goto out;
- zcache_eph_zbytes = atomic_long_sub_return(zsize,
- &zcache_eph_zbytes_atomic);
- zcache_eph_zpages = atomic_sub_return(zpages,
- &zcache_eph_zpages_atomic);
- zcache_evicted_eph_zpages += zpages;
- zcache_eph_pageframes =
- atomic_dec_return(&zcache_eph_pageframes_atomic);
- zcache_evicted_eph_pageframes++;
+ dec_zcache_eph_zbytes(zsize);
+ dec_zcache_eph_zpages(zpages);
+ inc_zcache_evicted_eph_zpages(zpages);
+ dec_zcache_eph_pageframes();
+ inc_zcache_evicted_eph_pageframes();
out:
return page;
}
@@ -991,6 +865,16 @@ out:
static atomic_t zcache_outstanding_writeback_pages_atomic = ATOMIC_INIT(0);
+static inline void inc_zcache_outstanding_writeback_pages(void)
+{
+ zcache_outstanding_writeback_pages =
+ atomic_inc_return(&zcache_outstanding_writeback_pages_atomic);
+}
+static inline void dec_zcache_outstanding_writeback_pages(void)
+{
+ zcache_outstanding_writeback_pages =
+ atomic_dec_return(&zcache_outstanding_writeback_pages_atomic);
+};
static void unswiz(struct tmem_oid oid, u32 index,
unsigned *type, pgoff_t *offset);
@@ -1004,8 +888,7 @@ static void unswiz(struct tmem_oid oid, u32 index,
static void zcache_end_swap_write(struct bio *bio, int err)
{
end_swap_bio_write(bio, err);
- zcache_outstanding_writeback_pages =
- atomic_dec_return(&zcache_outstanding_writeback_pages_atomic);
+ dec_zcache_outstanding_writeback_pages();
zcache_writtenback_pages++;
}
@@ -1120,8 +1003,7 @@ static int zcache_frontswap_writeback_zpage(int type, pgoff_t offset,
*/
(void)__swap_writepage(page, &wbc, zcache_end_swap_write);
page_cache_release(page);
- zcache_outstanding_writeback_pages =
- atomic_inc_return(&zcache_outstanding_writeback_pages_atomic);
+ inc_zcache_outstanding_writeback_pages();
return 0;
}
@@ -1357,9 +1239,9 @@ int zcache_put_page(int cli_id, int pool_id, struct tmem_oid *oidp,
if (pampd == NULL) {
ret = -ENOMEM;
if (ephemeral)
- zcache_failed_eph_puts++;
+ inc_zcache_failed_eph_puts();
else
- zcache_failed_pers_puts++;
+ inc_zcache_failed_pers_puts();
} else {
if (ramster_enabled)
ramster_do_preload_flnode(pool);
@@ -1369,7 +1251,7 @@ int zcache_put_page(int cli_id, int pool_id, struct tmem_oid *oidp,
}
zcache_put_pool(pool);
} else {
- zcache_put_to_flush++;
+ inc_zcache_put_to_flush();
if (ramster_enabled)
ramster_do_preload_flnode(pool);
if (atomic_read(&pool->obj_count) > 0)
@@ -1419,7 +1301,7 @@ int zcache_flush_page(int cli_id, int pool_id,
unsigned long flags;
local_irq_save(flags);
- zcache_flush_total++;
+ inc_zcache_flush_total();
pool = zcache_get_pool_by_id(cli_id, pool_id);
if (ramster_enabled)
ramster_do_preload_flnode(pool);
@@ -1429,7 +1311,7 @@ int zcache_flush_page(int cli_id, int pool_id,
zcache_put_pool(pool);
}
if (ret >= 0)
- zcache_flush_found++;
+ inc_zcache_flush_found();
local_irq_restore(flags);
return ret;
}
@@ -1442,7 +1324,7 @@ int zcache_flush_object(int cli_id, int pool_id,
unsigned long flags;
local_irq_save(flags);
- zcache_flobj_total++;
+ inc_zcache_flobj_total();
pool = zcache_get_pool_by_id(cli_id, pool_id);
if (ramster_enabled)
ramster_do_preload_flnode(pool);
@@ -1452,7 +1334,7 @@ int zcache_flush_object(int cli_id, int pool_id,
zcache_put_pool(pool);
}
if (ret >= 0)
- zcache_flobj_found++;
+ inc_zcache_flobj_found();
local_irq_restore(flags);
return ret;
}
@@ -1546,7 +1428,7 @@ static int zcache_local_new_pool(uint32_t flags)
int zcache_autocreate_pool(unsigned int cli_id, unsigned int pool_id, bool eph)
{
struct tmem_pool *pool;
- struct zcache_client *cli;
+ struct zcache_client *cli = NULL;
uint32_t flags = eph ? 0 : TMEM_POOL_PERSIST;
int ret = -1;
@@ -1615,7 +1497,7 @@ static void zcache_cleancache_put_page(int pool_id,
struct tmem_oid oid = *(struct tmem_oid *)&key;
if (!disable_cleancache_ignore_nonactive && !PageWasActive(page)) {
- zcache_eph_nonactive_puts_ignored++;
+ inc_zcache_eph_nonactive_puts_ignored();
return;
}
if (likely(ind == index))
@@ -1744,7 +1626,7 @@ static int zcache_frontswap_put_page(unsigned type, pgoff_t offset,
BUG_ON(!PageLocked(page));
if (!disable_frontswap_ignore_nonactive && !PageWasActive(page)) {
- zcache_pers_nonactive_puts_ignored++;
+ inc_zcache_pers_nonactive_puts_ignored();
ret = -ERANGE;
goto out;
}
@@ -1841,16 +1723,16 @@ struct frontswap_ops zcache_frontswap_register_ops(void)
static int __init enable_zcache(char *s)
{
- zcache_enabled = 1;
+ zcache_enabled = true;
return 1;
}
__setup("zcache", enable_zcache);
static int __init enable_ramster(char *s)
{
- zcache_enabled = 1;
+ zcache_enabled = true;
#ifdef CONFIG_RAMSTER
- ramster_enabled = 1;
+ ramster_enabled = true;
#endif
return 1;
}
@@ -1860,7 +1742,7 @@ __setup("ramster", enable_ramster);
static int __init no_cleancache(char *s)
{
- disable_cleancache = 1;
+ disable_cleancache = true;
return 1;
}
@@ -1868,7 +1750,7 @@ __setup("nocleancache", no_cleancache);
static int __init no_frontswap(char *s)
{
- disable_frontswap = 1;
+ disable_frontswap = true;
return 1;
}
@@ -1884,7 +1766,7 @@ __setup("nofrontswapexclusivegets", no_frontswap_exclusive_gets);
static int __init no_frontswap_ignore_nonactive(char *s)
{
- disable_frontswap_ignore_nonactive = 1;
+ disable_frontswap_ignore_nonactive = true;
return 1;
}
@@ -1892,7 +1774,7 @@ __setup("nofrontswapignorenonactive", no_frontswap_ignore_nonactive);
static int __init no_cleancache_ignore_nonactive(char *s)
{
- disable_cleancache_ignore_nonactive = 1;
+ disable_cleancache_ignore_nonactive = true;
return 1;
}
@@ -1900,8 +1782,8 @@ __setup("nocleancacheignorenonactive", no_cleancache_ignore_nonactive);
static int __init enable_zcache_compressor(char *s)
{
- strncpy(zcache_comp_name, s, ZCACHE_COMP_NAME_SZ);
- zcache_enabled = 1;
+ strlcpy(zcache_comp_name, s, sizeof(zcache_comp_name));
+ zcache_enabled = true;
return 1;
}
__setup("zcache=", enable_zcache_compressor);
@@ -1944,9 +1826,7 @@ static int __init zcache_init(void)
namestr = "ramster";
ramster_register_pamops(&zcache_pamops);
}
-#ifdef CONFIG_DEBUG_FS
zcache_debugfs_init();
-#endif
if (zcache_enabled) {
unsigned int cpu;
@@ -1986,7 +1866,7 @@ static int __init zcache_init(void)
old_ops = zcache_cleancache_register_ops();
pr_info("%s: cleancache enabled using kernel transcendent "
"memory and compression buddies\n", namestr);
-#ifdef ZCACHE_DEBUG
+#ifdef CONFIG_ZCACHE_DEBUG
pr_info("%s: cleancache: ignorenonactive = %d\n",
namestr, !disable_cleancache_ignore_nonactive);
#endif
@@ -2001,7 +1881,7 @@ static int __init zcache_init(void)
frontswap_tmem_exclusive_gets(true);
pr_info("%s: frontswap enabled using kernel transcendent "
"memory and compression buddies\n", namestr);
-#ifdef ZCACHE_DEBUG
+#ifdef CONFIG_ZCACHE_DEBUG
pr_info("%s: frontswap: excl gets = %d active only = %d\n",
namestr, frontswap_has_exclusive_gets,
!disable_frontswap_ignore_nonactive);
generated by cgit v1.2.3 (git 2.39.0) at 2024-07-19 11:21:26 +0300