From 293e9588b4264c41209c0daab6bcbf7b2479db0b Mon Sep 17 00:00:00 2001 From: michalbiesek Date: Wed, 1 Apr 2020 17:34:54 +0200 Subject: [PATCH 1/6] Add lipbmem to memkind configuration --- src/Makefile | 2 +- src/zmalloc.c | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Makefile b/src/Makefile index 7500d0a9383..362e2782ac5 100644 --- a/src/Makefile +++ b/src/Makefile @@ -193,7 +193,7 @@ endif ifeq ($(MALLOC),memkind) FINAL_CFLAGS+= -DUSE_MEMKIND - FINAL_LIBS+= -lmemkind -ldaxctl + FINAL_LIBS+= -lmemkind -ldaxctl -lpmem endif REDIS_CC=$(QUIET_CC)$(CC) $(FINAL_CFLAGS) diff --git a/src/zmalloc.c b/src/zmalloc.c index e204655d8c1..c045acea166 100644 --- a/src/zmalloc.c +++ b/src/zmalloc.c @@ -75,6 +75,7 @@ void zlibc_free(void *ptr) { #define dallocx(ptr,flags) je_dallocx(ptr,flags) #elif defined(USE_MEMKIND) #include +#include #define malloc(size) memkind_malloc(MEMKIND_DEFAULT,size) #define calloc(count,size) memkind_calloc(MEMKIND_DEFAULT,count,size) #define realloc_dram(ptr,size) memkind_realloc(MEMKIND_DEFAULT,ptr,size) From 7b9f79bada415ce32f9f6e61b9f48f7700b009eb Mon Sep 17 00:00:00 2001 From: michalbiesek Date: Wed, 1 Apr 2020 17:39:19 +0200 Subject: [PATCH 2/6] Add zmemcpy_pmem implementation --- src/zmalloc.c | 12 ++++++++++++ src/zmalloc.h | 1 + 2 files changed, 13 insertions(+) diff --git a/src/zmalloc.c b/src/zmalloc.c index c045acea166..f0b05725977 100644 --- a/src/zmalloc.c +++ b/src/zmalloc.c @@ -121,6 +121,14 @@ static void *zrealloc_pmem(void *ptr, size_t size) { zmalloc_pmem_not_available(); return NULL; } + +void *zmemcpy_pmem(void *dst, const void *src, size_t num) { + (void)(dst); + (void)(src); + (void)(num); + zmalloc_pmem_not_available(); + return NULL; +} #endif #define update_zmalloc_stat_alloc(__n) do { \ @@ -261,6 +269,10 @@ static void *zrealloc_pmem(void *ptr, size_t size) { return (char*)newptr+PREFIX_SIZE; #endif } + +void *zmemcpy_pmem(void *dst, const void *src, size_t num) { + return pmem_memcpy(dst, src, num, PMEM_F_MEM_NONTEMPORAL|PMEM_F_MEM_NODRAIN); +} #endif void *zmalloc(size_t size) { diff --git a/src/zmalloc.h b/src/zmalloc.h index 4b862839904..0a913ea312d 100644 --- a/src/zmalloc.h +++ b/src/zmalloc.h @@ -108,6 +108,7 @@ size_t zmalloc_get_threshold(void); void *zmalloc_dram(size_t size); void *zcalloc_dram(size_t size); void *zrealloc_dram(void *ptr, size_t size); +void *zmemcpy_pmem(void *dst, const void *src, size_t num); #ifdef HAVE_DEFRAG void zfree_no_tcache(void *ptr); From 79ab50cd8b34ff708d5099a0c8c1ea7e973b4e85 Mon Sep 17 00:00:00 2001 From: michalbiesek Date: Wed, 1 Apr 2020 23:05:36 +0200 Subject: [PATCH 3/6] Add zmemset_pmem implementation --- src/zmalloc.c | 12 ++++++++++++ src/zmalloc.h | 1 + 2 files changed, 13 insertions(+) diff --git a/src/zmalloc.c b/src/zmalloc.c index f0b05725977..eb5c4f26c7b 100644 --- a/src/zmalloc.c +++ b/src/zmalloc.c @@ -129,6 +129,14 @@ void *zmemcpy_pmem(void *dst, const void *src, size_t num) { zmalloc_pmem_not_available(); return NULL; } + +void *zmemset_pmem(void *ptr, int value, size_t num) { + (void)(ptr); + (void)(value); + (void)(num); + zmalloc_pmem_not_available(); + return NULL; +} #endif #define update_zmalloc_stat_alloc(__n) do { \ @@ -273,6 +281,10 @@ static void *zrealloc_pmem(void *ptr, size_t size) { void *zmemcpy_pmem(void *dst, const void *src, size_t num) { return pmem_memcpy(dst, src, num, PMEM_F_MEM_NONTEMPORAL|PMEM_F_MEM_NODRAIN); } + +void *zmemset_pmem(void *ptr, int value, size_t num) { + return pmem_memset(ptr, value, num, PMEM_F_MEM_NONTEMPORAL|PMEM_F_MEM_NODRAIN); +} #endif void *zmalloc(size_t size) { diff --git a/src/zmalloc.h b/src/zmalloc.h index 0a913ea312d..7f3943ea9fe 100644 --- a/src/zmalloc.h +++ b/src/zmalloc.h @@ -109,6 +109,7 @@ void *zmalloc_dram(size_t size); void *zcalloc_dram(size_t size); void *zrealloc_dram(void *ptr, size_t size); void *zmemcpy_pmem(void *dst, const void *src, size_t num); +void *zmemset_pmem(void *ptr, int value, size_t num); #ifdef HAVE_DEFRAG void zfree_no_tcache(void *ptr); From 99c1740c00f3efd9bcacee984c5df0812c1a4f9f Mon Sep 17 00:00:00 2001 From: michalbiesek Date: Tue, 28 Apr 2020 11:48:09 +0200 Subject: [PATCH 4/6] Expose zmalloc_pmem --- src/zmalloc.c | 4 ++-- src/zmalloc.h | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/zmalloc.c b/src/zmalloc.c index eb5c4f26c7b..9d69057d688 100644 --- a/src/zmalloc.c +++ b/src/zmalloc.c @@ -103,7 +103,7 @@ static void zfree_pmem(void *ptr) { zmalloc_pmem_not_available(); } -static void *zmalloc_pmem(size_t size) { +void *zmalloc_pmem(size_t size) { (void)(size); zmalloc_pmem_not_available(); return NULL; @@ -218,7 +218,7 @@ static void zfree_pmem(void *ptr) { #endif } -static void *zmalloc_pmem(size_t size) { +void *zmalloc_pmem(size_t size) { void *ptr = memkind_malloc(MEMKIND_DAX_KMEM, size+PREFIX_SIZE); if (!ptr && errno==ENOMEM) zmalloc_oom_handler(size); #ifdef HAVE_MALLOC_SIZE diff --git a/src/zmalloc.h b/src/zmalloc.h index 7f3943ea9fe..930678ea11a 100644 --- a/src/zmalloc.h +++ b/src/zmalloc.h @@ -108,6 +108,7 @@ size_t zmalloc_get_threshold(void); void *zmalloc_dram(size_t size); void *zcalloc_dram(size_t size); void *zrealloc_dram(void *ptr, size_t size); +void *zmalloc_pmem(size_t size); void *zmemcpy_pmem(void *dst, const void *src, size_t num); void *zmemset_pmem(void *ptr, int value, size_t num); From 53cc7e50a86370e23ce94a32ed071eca9dfdf1cf Mon Sep 17 00:00:00 2001 From: michalbiesek Date: Tue, 28 Apr 2020 11:51:10 +0200 Subject: [PATCH 5/6] Use pmem memset/memcpy function in sds implementation --- src/sds.c | 17 +++++++++++++---- src/sdsalloc.h | 2 ++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/sds.c b/src/sds.c index d7c657268f1..52f856110fd 100644 --- a/src/sds.c +++ b/src/sds.c @@ -44,6 +44,9 @@ const char *SDS_NOINIT = "SDS_NOINIT"; #define SDS_GENERAL_VARIANT 0 #define SDS_DRAM_VARIANT 1 +#define SDS_ON_PMEM 0 +#define SDS_ON_DRAM 1 + static inline int sdsHdrSize(char type) { switch(type&SDS_TYPE_MASK) { case SDS_TYPE_5: @@ -93,18 +96,24 @@ static sds _sdsnewlen(const void *init, size_t initlen, int on_dram) { void *sh; sds s; char type = sdsReqType(initlen); + int sds_place; + /* Empty strings are usually created in order to append. Use type 8 * since type 5 is not good at this. */ if (type == SDS_TYPE_5 && initlen == 0) type = SDS_TYPE_8; int hdrlen = sdsHdrSize(type); unsigned char *fp; /* flags pointer. */ + if ((on_dram == SDS_GENERAL_VARIANT) && (hdrlen+initlen+1) >= zmalloc_get_threshold()) { + sds_place = SDS_ON_PMEM; + } else { + sds_place = SDS_ON_DRAM; + } - sh = (on_dram == SDS_DRAM_VARIANT) ? s_dram_malloc(hdrlen+initlen+1) - : s_malloc(hdrlen+initlen+1); + sh = (sds_place == SDS_ON_DRAM) ? s_dram_malloc(hdrlen+initlen+1) : s_pmem_malloc(hdrlen+initlen+1); if (init==SDS_NOINIT) init = NULL; else if (!init) - memset(sh, 0, hdrlen+initlen+1); + (sds_place == SDS_ON_DRAM) ? memset(sh, 0, hdrlen+initlen+1) : zmemset_pmem(sh, 0, hdrlen+initlen+1); if (sh == NULL) return NULL; s = (char*)sh+hdrlen; fp = ((unsigned char*)s)-1; @@ -143,7 +152,7 @@ static sds _sdsnewlen(const void *init, size_t initlen, int on_dram) { } } if (initlen && init) - memcpy(s, init, initlen); + (sds_place == SDS_ON_DRAM) ? memcpy(s, init, initlen) : zmemcpy_pmem(s, init, initlen); s[initlen] = '\0'; return s; } diff --git a/src/sdsalloc.h b/src/sdsalloc.h index 7c7a6a80f09..4108ec98cd2 100644 --- a/src/sdsalloc.h +++ b/src/sdsalloc.h @@ -41,6 +41,8 @@ #include "zmalloc.h" #define s_malloc zmalloc +#define s_dram_malloc zmalloc_dram +#define s_pmem_malloc zmalloc_pmem #define s_realloc zrealloc #define s_free zfree #define s_dram_malloc zmalloc_dram From 176005f6719d93da9323c287eadf333fefc65910 Mon Sep 17 00:00:00 2001 From: michalbiesek Date: Tue, 28 Apr 2020 11:59:34 +0200 Subject: [PATCH 6/6] Move setting s[initlen] to avoid cache miss --- src/sds.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sds.c b/src/sds.c index 52f856110fd..e8d998a0b6e 100644 --- a/src/sds.c +++ b/src/sds.c @@ -151,9 +151,10 @@ static sds _sdsnewlen(const void *init, size_t initlen, int on_dram) { break; } } + s[initlen] = '\0'; if (initlen && init) (sds_place == SDS_ON_DRAM) ? memcpy(s, init, initlen) : zmemcpy_pmem(s, init, initlen); - s[initlen] = '\0'; + return s; }