finally fix the old memory leak in yajl_tree_parse()

- use wrapped malloc() et al wrappers consistently
- update example/parse_config.c to do memory leak detection
- add a regression test using example/parse_config

Several issues in lloyd/yajl complained about this leak, and comments in
lloyd/yajl#102 showed a mostly correct fix though none of these issues
mentioned or actually fixed the directly related error reporting
problem.

Fixes lloyd/yajl#102, fixes lloyd/yajl#113, fixes lloyd/yajl#168, fixes
lloyd/yajl#191, fixes lloyd/yajl#223, fixes lloyd/yajl#250.  Also fixes
lloy/yajl#185.

example/Makefile

@@ -9,12 +9,19 @@ MAN =	# empty
 # don't install
 proginstall:
 
+# xxx hmmm....  stdout and stderr are reversed
+#
+T1 = [ $$(expr "$$(${.OBJDIR}/${PROG} < ${.CURDIR}/sample.config 2>&1)" : "memory leaks:.0.Logging/timeFormat: utc") -eq 39 ]
+T2 = [ $$(expr "$$(echo '{broken:' | ${.OBJDIR}/${PROG} 2>&1)" : "tree_parse_error: lexical error: invalid char in json text.*memory leaks:.0") -eq 165 ]
+
 regress:
 .if defined(USE_ASAN)
-	if [  -x /usr/sbin/paxctl ]; then /usr/sbin/paxctl +a ./parse_config; fi
-	ulimit -v unlimited && [ "$$(${.OBJDIR}/${PROG} < ${.CURDIR}/sample.config)" = "Logging/timeFormat: utc" ]
+	if [  -x /usr/sbin/paxctl ]; then /usr/sbin/paxctl +a ${.OBJDIR}/${PROG}; fi
+	ulimit -v unlimited && ${T1}
+	ulimit -v unlimited && ${T2}
 .else
-	[ "$$(${.OBJDIR}/${PROG} < ${.CURDIR}/sample.config)" = "Logging/timeFormat: utc" ]
+	${T1}
+	${T2}
 .endif
 
 .include <bsd.prog.mk>

example/parse_config.c

@@ -15,11 +15,78 @@
  */
 
 #include <stdio.h>
+#include <stdbool.h>
+#include <stdlib.h>
 #include <string.h>
 
+#include <assert.h>
+
 #include "yajl/yajl_tree.h"
 
-static unsigned char fileData[65536];
+/* context storage for memory debugging routines */
+typedef struct
+{
+    bool do_printfs;
+    unsigned int numFrees;
+    unsigned int numMallocs;
+    /* XXX: we really need a hash table here with per-allocation
+     *      information to find any missing free() calls */
+} yajlTestMemoryContext;
+
+/* cast void * into context */
+#define TEST_CTX(vptr) ((yajlTestMemoryContext *) (vptr))
+
+static void
+yajlTestFree(void *ctx,
+             void *ptr)
+{
+    assert(ptr != NULL);
+    TEST_CTX(ctx)->numFrees++;
+    if (TEST_CTX(ctx)->do_printfs) {
+        fprintf(stderr, "yfree:  %p\n", ptr);
+    }
+    free(ptr);
+}
+
+static void *
+yajlTestMalloc(void *ctx,
+               size_t sz)
+{
+    void *rv = NULL;
+
+    assert(sz != 0);
+    TEST_CTX(ctx)->numMallocs++;
+    rv = malloc(sz);
+    assert(rv != NULL);
+    if (TEST_CTX(ctx)->do_printfs) {
+        fprintf(stderr, "yalloc:  %p of %ju\n", rv, sz);
+    }
+    return rv;
+}
+
+static void *
+yajlTestRealloc(void *ctx,
+                void *ptr,
+                size_t sz)
+{
+    void *rv = NULL;
+
+    if (ptr == NULL) {
+        assert(sz != 0);
+        TEST_CTX(ctx)->numMallocs++;
+    } else if (sz == 0) {
+        TEST_CTX(ctx)->numFrees++;
+    }
+    rv = realloc(ptr, sz);
+    assert(rv != NULL);
+    if (TEST_CTX(ctx)->do_printfs) {
+        fprintf(stderr, "yrealloc:  %p -> %p of %ju\n", ptr, rv, sz);
+    }
+    return rv;
+}
+
+
+static unsigned char fileData[65536];   /* xxx: allocate to size of file, error if stdin can't be stat()ed? */
 
 int
 main(void)
@@ -28,35 +95,49 @@ main(void)
     yajl_val node;
     char errbuf[1024];
 
-    /* NUL plug the buffers */
-    fileData[0] = '\0';
-    errbuf[0] = '\0';
+    /* memory allocation debugging: allocate a structure which holds
+     * allocation routines */
+    yajl_alloc_funcs allocFuncs = {
+        yajlTestMalloc,
+        yajlTestRealloc,
+        yajlTestFree,
+        (void *) NULL
+    };
+
+    /* memory allocation debugging: allocate a structure which collects
+     * statistics */
+    yajlTestMemoryContext memCtx;
+
+    memCtx.do_printfs = false;          /* xxx set from a command option */
+    memCtx.numMallocs = 0;
+    memCtx.numFrees = 0;
+
+    allocFuncs.ctx = (void *) &memCtx;
+    yajl_tree_parse_afs = &allocFuncs;
 
     /* read the entire config file */
     rd = fread((void *) fileData, (size_t) 1, sizeof(fileData) - 1, stdin);
 
     /* file read error handling */
-    if (rd == 0 && !feof(stdin)) {
-        fprintf(stderr, "error encountered on file read\n");
-        return 1;
-    } else if (rd >= sizeof(fileData) - 1) {
+    if ((rd == 0 && !feof(stdin)) || ferror(stdin)) {
+        perror("error encountered on file read");
+        exit(1);
+    } else if (!feof(stdin)) {
         fprintf(stderr, "config file too big\n");
-        return 1;
+        exit(1);
     }
+    fileData[rd] = '\0';
 
     /* we have the whole config file in memory.  let's parse it ... */
     node = yajl_tree_parse((const char *) fileData, errbuf, sizeof(errbuf));
 
     /* parse error handling */
     if (node == NULL) {
-        fprintf(stderr, "parse_error: ");
-        if (strlen(errbuf)) {
-            fprintf(stderr, "%s", errbuf);
-        } else {
-            fprintf(stderr, "unknown error");
-        }
-        fprintf(stderr, "\n");
-        return 1;
+        assert(errbuf != NULL);
+        fprintf(stderr, "tree_parse_error: %s\n", errbuf);
+        fprintf(stderr, "memory leaks:\t%u\n", memCtx.numMallocs - memCtx.numFrees);
+
+        exit(1);
     }
 
     /* ... and extract a nested value from the config file */
@@ -74,5 +155,7 @@ main(void)
 
     yajl_tree_free(node);
 
-    return 0;
+    fprintf(stderr, "memory leaks:\t%u\n", memCtx.numMallocs - memCtx.numFrees);
+
+    exit(memCtx.numMallocs - memCtx.numFrees ? 1 : 0);
 }

src/Makefile

@@ -166,7 +166,7 @@ install:	includes
 incinstall: .PHONY
 .endif
 
-.if ${TARGET_OSNAME} == "Linux"
+.if defined(TARGET_OSNAME) && ${TARGET_OSNAME} == "Linux"
 # XXX stupid GNU BinUtils LD changed its command line syntax recently,
 # apparently without concern for backward compatability.
 #

src/yajl/yajl_tree.h

@@ -37,6 +37,9 @@
 extern "C" {
 #endif
 
+/*+ an optional hook to allow use of custom yajl_alloc_funcs with yajl_tree_parse() +*/
+extern yajl_alloc_funcs *yajl_tree_parse_afs;
+
 /*+ possible data types that a yajl_val_s can hold +*/
 typedef enum {
     yajl_t_string = 1,

src/yajl_parser.c

@@ -158,27 +158,34 @@ yajl_status
 yajl_do_finish(yajl_handle hand)
 {
     yajl_status stat;
-    stat = yajl_do_parse(hand,(const unsigned char *) " ",(size_t) 1);
+    size_t offset = hand->bytesConsumed;
 
-    if (stat != yajl_status_ok) return stat;
+    stat = yajl_do_parse(hand, (const unsigned char *) " ", (size_t) 1);
+    hand->bytesConsumed = offset;
 
-    switch(yajl_bs_current(hand->stateStack))
-    {
-        case yajl_state_parse_error:
-        case yajl_state_lexical_error:
-            return yajl_status_error;
-        case yajl_state_got_value:
-        case yajl_state_parse_complete:
-            return yajl_status_ok;
-        default:
-            if (!(hand->flags & yajl_allow_partial_values))
-            {
-                yajl_bs_set(hand->stateStack, yajl_state_parse_error);
-                hand->parseError = "premature EOF";
-                return yajl_status_error;
-            }
-            return yajl_status_ok;
+    if (stat != yajl_status_ok) {
+        return stat;
+    }
+
+    switch (yajl_bs_current(hand->stateStack)) {
+    case yajl_state_parse_error:
+    case yajl_state_lexical_error:
+        return yajl_status_error;
+
+    case yajl_state_got_value:
+    case yajl_state_parse_complete:
+        return yajl_status_ok;
+
+    default:
+        break;
+    }
+    if (!(hand->flags & yajl_allow_partial_values)) {
+        yajl_bs_set(hand->stateStack, yajl_state_parse_error);
+        hand->parseError = "premature EOF";
+
+        return yajl_status_error;
     }
+    return yajl_status_ok;
 }
 
 yajl_status