diff --git a/Makefile b/Makefile index bfebac5..a509f29 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,7 @@ ALL_CFLAGS=$(CFLAGS) -std=c99 -Wall -Wextra -Wpedantic OBJ=\ build.o\ deps.o\ + dyndep.o\ env.o\ graph.o\ htab.o\ diff --git a/build.c b/build.c index 83d51b7..58ce586 100644 --- a/build.c +++ b/build.c @@ -13,6 +13,7 @@ #include #include "build.h" #include "deps.h" +#include "dyndep.h" #include "env.h" #include "graph.h" #include "log.h" @@ -105,6 +106,10 @@ queue(struct edge *e) { struct edge **front = &work; + if (e->flags & FLAG_QUEUED) + return; + e->flags |= FLAG_QUEUED; + if (e->pool && e->rule != &phonyrule) { if (e->pool->numjobs == e->pool->maxjobs) front = &e->pool->work; @@ -115,13 +120,89 @@ queue(struct edge *e) *front = e; } +static void +computedirty(struct edge *e, struct node *newest) +{ + struct node *n; + size_t i; + bool generator, restat; + + /* all outputs are dirty if any are older than the newest input */ + generator = edgevarbool(e, "generator"); + restat = edgevarbool(e, "restat"); + for (i = 0; i < e->nout && !(e->flags & FLAG_DIRTY_OUT); ++i) { + n = e->out[i]; + if (isdirty(n, newest, generator, restat)) { + n->dirty = true; + e->flags |= FLAG_DIRTY_OUT; + } + } + if (e->flags & FLAG_DIRTY) { + for (i = 0; i < e->nout; ++i) { + n = e->out[i]; + if (buildopts.explain && !n->dirty) { + if (e->flags & FLAG_DIRTY_IN) + warn("explain %s: input is dirty", n->path->s); + else if (e->flags & FLAG_DIRTY_OUT) + warn("explain %s: output of generating action is dirty", n->path->s); + } + n->dirty = true; + } + } +} + +void +buildupdate(struct node *n) +{ + struct edge *e; + struct node *newest; + bool planned; + size_t i; + + e = n->gen; + if (e->flags & FLAG_CYCLE) + fatal("dependency cycle involving '%s'", n->path->s); + e->flags |= FLAG_CYCLE | FLAG_WORK; + /* if the edge is already marked dirty ntotal was already updated */ + planned = e->flags & FLAG_DIRTY; + for (i = e->outdynidx; i < e->nout; ++i) { + n = e->out[i]; + if (n->mtime == MTIME_UNKNOWN) { + n->dirty = false; + nodestat(n); + } + } + newest = NULL; + for (i = e->indynidx; i < e->inorderidx; ++i) { + n = e->in[i]; + buildadd(n); + if (i < e->inorderidx) { + if (n->dirty) + e->flags |= FLAG_DIRTY_IN; + if (n->mtime != MTIME_MISSING && !isnewer(newest, n)) + newest = n; + } + if (n->dirty || (n->gen && n->gen->nblock > 0)) + ++e->nblock; + } + computedirty(e, newest); + if (!(e->flags & FLAG_DIRTY_OUT)) + e->nprune = e->nblock; + if (e->flags & FLAG_DIRTY) { + if (e->nblock == 0) + queue(e); + if (!planned && e->rule != &phonyrule) + ++ntotal; + } + e->flags &= ~FLAG_CYCLE; +} + void buildadd(struct node *n) { struct edge *e; struct node *newest; size_t i; - bool generator, restat; e = n->gen; if (!e) { @@ -158,28 +239,7 @@ buildadd(struct node *n) if (n->dirty || (n->gen && n->gen->nblock > 0)) ++e->nblock; } - /* all outputs are dirty if any are older than the newest input */ - generator = edgevar(e, "generator", true); - restat = edgevar(e, "restat", true); - for (i = 0; i < e->nout && !(e->flags & FLAG_DIRTY_OUT); ++i) { - n = e->out[i]; - if (isdirty(n, newest, generator, restat)) { - n->dirty = true; - e->flags |= FLAG_DIRTY_OUT; - } - } - if (e->flags & FLAG_DIRTY) { - for (i = 0; i < e->nout; ++i) { - n = e->out[i]; - if (buildopts.explain && !n->dirty) { - if (e->flags & FLAG_DIRTY_IN) - warn("explain %s: input is dirty", n->path->s); - else if (e->flags & FLAG_DIRTY_OUT) - warn("explain %s: output of generating action is dirty", n->path->s); - } - n->dirty = true; - } - } + computedirty(e, newest); if (!(e->flags & FLAG_DIRTY_OUT)) e->nprune = e->nblock; if (e->flags & FLAG_DIRTY) { @@ -189,6 +249,8 @@ buildadd(struct node *n) ++ntotal; } e->flags &= ~FLAG_CYCLE; + if (e->dyndep) + dyndepload(e->dyndep, false); } static size_t @@ -351,6 +413,13 @@ nodedone(struct node *n, bool prune) struct edge *e; size_t i, j; + /* mark node clean for computedirty of edges with dyndeps */ + n->dirty = false; + + /* if this node is a dyndep it can be loaded now */ + if (n->dyndep) + dyndepload(n->dyndep, false); + for (i = 0; i < n->nuse; ++i) { e = n->use[i]; /* skip edges not used in this build */ @@ -370,43 +439,45 @@ nodedone(struct node *n, bool prune) } } -static bool -shouldprune(struct edge *e, struct node *n, int64_t old) -{ - struct node *in, *newest; - size_t i; - - if (old != n->mtime) - return false; - newest = NULL; - for (i = 0; i < e->inorderidx; ++i) { - in = e->in[i]; - nodestat(in); - if (in->mtime != MTIME_MISSING && !isnewer(newest, in)) - newest = in; - } - if (newest) - n->logmtime = newest->mtime; - - return true; -} - static void edgedone(struct edge *e) { - struct node *n; + struct node *n, *newest; size_t i; struct string *rspfile; - bool restat; + bool restat, prune, pruned; int64_t old; - restat = edgevar(e, "restat", true); + /* mark edge clean for dyndepload */ + e->flags &= ~FLAG_DIRTY; + + newest = NULL; + prune = pruned = false; + restat = edgevarbool(e, "restat"); for (i = 0; i < e->nout; ++i) { n = e->out[i]; old = n->mtime; nodestat(n); n->logmtime = n->mtime == MTIME_MISSING ? 0 : n->mtime; - nodedone(n, restat && shouldprune(e, n, old)); + + prune = restat && old != n->mtime; + pruned = pruned || prune; + nodedone(n, prune); + } + /* if any output was pruned, find the newest input non-order-only + * input and record it for each output of this edge */ + if (pruned) { + for (i = 0; i < e->inorderidx; ++i) { + n = e->in[i]; + nodestat(n); + if (n->mtime != MTIME_MISSING && !isnewer(newest, n)) + newest = n; + } + } + for (i = 0; i < e->nout; ++i) { + n = e->out[i]; + if (newest) + n->logmtime = newest->mtime; } rspfile = edgevar(e, "rspfile", false); if (rspfile && !buildopts.keeprsp) diff --git a/build.h b/build.h index 1c52b7e..d9b8607 100644 --- a/build.h +++ b/build.h @@ -12,5 +12,7 @@ extern struct buildoptions buildopts; void buildreset(void); /* schedule a particular target to be built */ void buildadd(struct node *); +/* reschedule a particular target to be built */ +void buildupdate(struct node *); /* execute rules to build the scheduled targets */ void build(void); diff --git a/build.ninja b/build.ninja index 1edea67..4a12e75 100644 --- a/build.ninja +++ b/build.ninja @@ -10,6 +10,7 @@ rule link build build.o: cc build.c build deps.o: cc deps.c +build dyndep.o: cc dyndep.c build env.o: cc env.c build graph.o: cc graph.c build htab.o: cc htab.c @@ -20,6 +21,6 @@ build scan.o: cc scan.c build tool.o: cc tool.c build tree.o: cc tree.c build util.o: cc util.c -build samu: link build.o deps.o env.o graph.o htab.o log.o parse.o samu.o scan.o tool.o tree.o util.o +build samu: link build.o deps.o dyndep.o env.o graph.o htab.o log.o parse.o samu.o scan.o tool.o tree.o util.o default samu diff --git a/dyndep.c b/dyndep.c new file mode 100644 index 0000000..8675d20 --- /dev/null +++ b/dyndep.c @@ -0,0 +1,271 @@ +#include +#include +#include +#include +#include +#include +#include +#include "build.h" +#include "dyndep.h" +#include "env.h" +#include "graph.h" +#include "htab.h" +#include "scan.h" +#include "util.h" + +struct nodearray { + struct node **node; + size_t len; +}; + +const char *dyndepversion = "1.0"; + +static struct dyndep *alldyndeps; + +void +dyndepinit(void) +{ + struct dyndep *d; + + /* delete old dyndeps in case we rebuilt the manifest */ + while (alldyndeps) { + d = alldyndeps; + alldyndeps = d->allnext; + free(d->use); + free(d); + } +} + +struct dyndep * +mkdyndep(struct node *n) +{ + struct dyndep *d; + + if (n->dyndep) + return n->dyndep; + + d = xmalloc(sizeof(*d)); + d->node = n; + d->use = NULL; + d->nuse = 0; + d->update = NULL; + d->nupdate = 0; + d->done = false; + d->allnext = alldyndeps; + alldyndeps = d; + n->dyndep = d; + + return d; +} + +void +dyndepuse(struct dyndep *d, struct edge *e) +{ + e->dyndep = d; + + /* allocate in powers of two */ + if (!(d->nuse & (d->nuse - 1))) + d->use = xreallocarray(d->use, d->nuse ? d->nuse * 2 : 1, sizeof(e)); + d->use[d->nuse++] = e; +} + +static void +dyndepupdate(struct dyndep *d, struct node *n) +{ + /* allocate in powers of two */ + if (!(d->nupdate & (d->nupdate - 1))) + d->update = xreallocarray(d->update, d->nupdate ? d->nupdate * 2 : 1, sizeof(n)); + d->update[d->nupdate++] = n; +} + +static void +parselet(struct scanner *s, struct evalstring **val) +{ + scanchar(s, '='); + *val = scanstring(s, false); + scannewline(s); +} + +static void +checkversion(const char *ver) +{ + int nmajor, nminor = 0; + if ((sscanf(ver, "%d.%d", &nmajor, &nminor) < 1) || (nmajor > 1)) + fatal("ninja_dyndep_version %s is newer than %s", ver, dyndepversion); +} + +static void +dyndepparseedge(struct scanner *s, struct environment *env) +{ + struct dyndep *d; + struct edge *e; + struct node *n; + static struct nodearray nodes; + static size_t nodescap; + struct evalstring *str; + char *name; + struct string *val; + + str = scanstring(s, true); + if (!str) + scanerror(s, "expected explicit output"); + + val = enveval(env, str); + n = nodeget(val->s, val->n); + if (!n) + fatal("unknown target '%s'", val->s); + e = n->gen; + if (!e) + fatal("no action to generate '%s' in dyndep", val->s); + free(val); + d = e->dyndep; + if (!d) + fatal("target '%s' does not mention this dyndep", n->path->s); + + dyndepupdate(d, n); + + nodes.len = 0; + if (scanpipe(s, 1)) { + for (; (str = scanstring(s, true)); ) { + val = enveval(e->env, str); + canonpath(val); + if (nodes.len == nodescap) { + nodescap = nodes.node ? nodescap * 2 : 32; + nodes.node = xreallocarray(nodes.node, nodescap, sizeof(nodes.node[0])); + } + nodes.node[nodes.len++] = mknode(val); + } + } + if (nodes.len) + edgeadddynouts(e, nodes.node, nodes.len); + + scanchar(s, ':'); + name = scanname(s); + if (strcmp(name, "dyndep") != 0) + fatal("unexpected rule '%s' in dyndep", name); + free(name); + + nodes.len = 0; + if (scanpipe(s, 1)) { + for (; (str = scanstring(s, true)); ) { + val = enveval(e->env, str); + canonpath(val); + if (nodes.len == nodescap) { + nodescap = nodes.node ? nodescap * 2 : 32; + nodes.node = xreallocarray(nodes.node, nodescap, sizeof(nodes.node[0])); + } + nodes.node[nodes.len++] = mknode(val); + } + } + if (nodes.len) + edgeadddyndeps(e, nodes.node, nodes.len); + + scannewline(s); + while (scanindent(s)) { + name = scanname(s); + if (strcmp(name, "restat") != 0) + fatal("unexpected variable '%s' in dyndep", name); + parselet(s, &str); + envaddvar(e->env, "restat", enveval(env, str)); + } + + e->flags |= FLAG_DYNDEP; +} + +static void +dyndepparse(const char *name) +{ + struct scanner s; + struct environment *env; + struct evalstring *str; + struct string *val; + bool version = false; + char *var; + + scaninit(&s, name); + + env = mkenv(NULL); + for (;;) { + switch (scankeyword(&s, &var)) { + case BUILD: + if (!version) + goto version; + dyndepparseedge(&s, env); + break; + case VARIABLE: + if (version) + scanerror(&s, "unexpected variable '%s'", var); + parselet(&s, &str); + val = enveval(env, str); + if (strcmp(var, "ninja_dyndep_version") == 0) + checkversion(val->s); + envaddvar(env, var, val); + version = true; + break; + case EOF: + if (!version) + goto version; + scanclose(&s); + return; + default: + scanerror(&s, "unexpected keyword"); + } + } +version: + scanerror(&s, "expected 'ninja_dyndep_version'"); +} + +bool +dyndepload(struct dyndep *d, bool prune) +{ + const char *name; + struct edge *e; + struct node *n; + size_t i; + + if (d->done) + return true; + + name = d->node->path->s; + + if (!d->node->gen) + fatal("dyndep not created by any action: '%s'", name); + + if (!prune) { + /* only load dyndep file if its clean and nothing is blocking it */ + if (d->node->gen->flags & FLAG_DIRTY || d->node->gen->nblock > 0) + return false; + } else { + nodestat(d->node); + if (d->node->mtime == MTIME_MISSING) + return false; + } + + if (buildopts.explain) + warn("loading dyndep file: '%s'", name); + + dyndepparse(name); + + d->done = true; + + if (prune) + return true; + + /* verify that all nodes with this dyndep are loaded */ + for (i = 0; i < d->nuse; i++) { + e = d->use[i]; + if (!(e->flags & FLAG_DYNDEP)) + fatal("target '%s' not mentioned in dyndep file '%s'", + e->out[0]->path->s, d->node->path->s); + } + /* (re)-schedule the nodes updated by this dyndep */ + for (i = 0; i < d->nupdate; i++) { + n = d->update[i]; + /* only update the node if it was previously added to the build */ + if (n->gen->flags & FLAG_WORK) + buildupdate(n); + n->gen->flags &= ~FLAG_DYNDEP; + } + + return true; +} diff --git a/dyndep.h b/dyndep.h new file mode 100644 index 0000000..d4aa2bd --- /dev/null +++ b/dyndep.h @@ -0,0 +1,29 @@ +struct node; + +struct dyndep { + /* the node building the dyndep file */ + struct node *node; + + /* edges using this dyndep */ + struct edge **use; + size_t nuse; + + /* nodes this dyndep updated */ + struct node **update; + size_t nupdate; + + /* is this dyndep file already loaded */ + _Bool done; + + /* used for alldyndeps linked list */ + struct dyndep *allnext; +}; + +void dyndepinit(void); + +/* create a new dyndep or return existing dyndep */ +struct dyndep *mkdyndep(struct node *); +/* load the dyndep */ +bool dyndepload(struct dyndep *, bool); +/* record the usage of a dyndep by an edge */ +void dyndepuse(struct dyndep *, struct edge *); diff --git a/env.c b/env.c index d333e7e..cd68964 100644 --- a/env.c +++ b/env.c @@ -246,6 +246,19 @@ edgevar(struct edge *e, char *var, bool escape) return merge(str, n); } +bool +edgevarbool(struct edge *e, char *var) +{ + struct string *val; + bool rv; + + val = edgevar(e, var, true); + if (!val) + return false; + rv = val->n > 0; + return rv; +} + static void addpool(struct pool *p) { diff --git a/env.h b/env.h index 37f4dd4..5ff112a 100644 --- a/env.h +++ b/env.h @@ -42,6 +42,9 @@ struct pool *poolget(char *); /* evaluate and return an edge's variable, optionally shell-escaped */ struct string *edgevar(struct edge *, char *, _Bool); +/* evaluate and return an edge's boolean variable */ +_Bool edgevarbool(struct edge *, char *); + extern struct environment *rootenv; extern struct rule phonyrule; extern struct pool consolepool; diff --git a/graph.c b/graph.c index 73523be..06ca0a2 100644 --- a/graph.c +++ b/graph.c @@ -58,6 +58,7 @@ mknode(struct string *path) n = xmalloc(sizeof(*n)); n->path = path; n->shellpath = NULL; + n->dyndep = NULL; n->gen = NULL; n->use = NULL; n->nuse = 0; @@ -152,6 +153,7 @@ mkedge(struct environment *parent) e = xmalloc(sizeof(*e)); e->env = mkenv(parent); + e->dyndep = NULL; e->pool = NULL; e->out = NULL; e->nout = 0; @@ -227,3 +229,43 @@ edgeadddeps(struct edge *e, struct node **deps, size_t ndeps) e->inorderidx += ndeps; e->nin += ndeps; } + +void +edgeadddyndeps(struct edge *e, struct node **deps, size_t ndeps) +{ + struct node **order, *n; + size_t norder, i; + + for (i = 0; i < ndeps; ++i) { + n = deps[i]; + if (!n->gen) + n->gen = mkphony(n); + nodeuse(n, e); + } + e->indynidx = e->inorderidx; + e->in = xreallocarray(e->in, e->nin + ndeps, sizeof(e->in[0])); + order = e->in + e->inorderidx; + norder = e->nin - e->inorderidx; + memmove(order + ndeps, order, norder * sizeof(e->in[0])); + memcpy(order, deps, ndeps * sizeof(e->in[0])); + e->inorderidx += ndeps; + e->nin += ndeps; +} + +void +edgeadddynouts(struct edge *e, struct node **outs, size_t nouts) +{ + struct node *n; + size_t i; + + for (i = 0; i < nouts; ++i) { + n = outs[i]; + if (n->gen && n->gen->rule != &phonyrule) + fatal("multiple rules generate '%s'", n->path->s); + n->gen = e; + } + e->outdynidx = e->nout; + e->out = xreallocarray(e->out, e->nout + nouts, sizeof(e->out[0])); + memcpy(e->out + e->nout, outs, nouts * sizeof(e->out[0])); + e->nout += nouts; +} diff --git a/graph.h b/graph.h index 4d00a9b..92570fc 100644 --- a/graph.h +++ b/graph.h @@ -19,6 +19,9 @@ struct node { struct edge *gen, **use; size_t nuse; + /* dyndep or NULL if the node is not a dyndep */ + struct dyndep *dyndep; + /* command hash used to build this output, read from build log */ uint64_t hash; @@ -39,10 +42,13 @@ struct edge { struct node **out, **in; size_t nout, nin; - /* index of first implicit output */ - size_t outimpidx; - /* index of first implicit and order-only input */ - size_t inimpidx, inorderidx; + /* dyndep or NULL if the edge has no dyndep */ + struct dyndep *dyndep; + + /* index of first implicit adn dyndep output */ + size_t outimpidx, outdynidx; + /* index of first implicit, dyndep and order-only input */ + size_t inimpidx, indynidx, inorderidx; /* command hash */ uint64_t hash; @@ -60,6 +66,8 @@ struct edge { FLAG_DIRTY = FLAG_DIRTY_IN | FLAG_DIRTY_OUT, FLAG_CYCLE = 1 << 5, /* used for cycle detection */ FLAG_DEPS = 1 << 6, /* dependencies loaded */ + FLAG_DYNDEP = 1 << 7, /* dyndep loaded */ + FLAG_QUEUED = 1 << 8, /* edge is queued */ } flags; /* used to coordinate ready work in build() */ @@ -87,6 +95,10 @@ struct edge *mkedge(struct environment *parent); void edgehash(struct edge *); /* add dependencies from $depfile or .ninja_deps as implicit inputs */ void edgeadddeps(struct edge *e, struct node **deps, size_t ndeps); +/* add dependencies from dyndep files as implicit inputs */ +void edgeadddyndeps(struct edge *e, struct node **deps, size_t ndeps); +/* add outputs from dyndep as implicit outputs */ +void edgeadddynouts(struct edge *e, struct node **outs, size_t nouts); /* a single linked list of all edges, valid up until build() */ extern struct edge *alledges; diff --git a/log.c b/log.c index ab1dc29..f6a5125 100644 --- a/log.c +++ b/log.c @@ -38,6 +38,7 @@ loginit(const char *builddir) size_t sz = 0, nline, nentry, i; struct edge *e; struct node *n; + struct string *path; int64_t mtime; nline = 0; @@ -80,8 +81,12 @@ loginit(const char *builddir) if (!s) continue; n = nodeget(s, 0); - if (!n || !n->gen) - continue; + if (!n) { + path = mkstr(strlen(s)); + memcpy(path->s, s, path->n); + path->s[path->n] = '\0'; + n = mknode(path); + } if (n->logmtime == MTIME_MISSING) ++nentry; n->logmtime = mtime; diff --git a/parse.c b/parse.c index 932a573..d32a0a5 100644 --- a/parse.c +++ b/parse.c @@ -3,13 +3,14 @@ #include #include #include "env.h" +#include "dyndep.h" #include "graph.h" #include "parse.h" #include "scan.h" #include "util.h" struct parseoptions parseopts; -const char *ninjaversion = "1.9.0"; +const char *ninjaversion = "1.10.0"; struct node **deftarg; size_t ndeftarg; @@ -71,7 +72,7 @@ parseedge(struct scanner *s, struct environment *env) struct edge *e; struct evalstring *out, *in, *str, **end; char *name; - struct string *val; + struct string *val, *dyndep; struct node *n; size_t i; int p; @@ -102,7 +103,7 @@ parseedge(struct scanner *s, struct environment *env) pushstr(&end, str); p = scanpipe(s, 2); } - e->inorderidx = e->nin; + e->indynidx = e->inorderidx = e->nin; if (p == 2) { for (; (str = scanstring(s, true)); ++e->nin) pushstr(&end, str); @@ -134,6 +135,9 @@ parseedge(struct scanner *s, struct environment *env) ++i; } } + e->outdynidx = e->nout; + + dyndep = edgevar(e, "dyndep", true); e->in = xreallocarray(NULL, e->nin, sizeof(e->in[0])); for (i = 0; i < e->nin; in = str, ++i) { @@ -143,8 +147,15 @@ parseedge(struct scanner *s, struct environment *env) n = mknode(val); e->in[i] = n; nodeuse(n, e); + if (dyndep && strcmp(n->path->s, dyndep->s) == 0) { + dyndepuse(mkdyndep(n), e); + dyndep = NULL; + } } + if (dyndep) + fatal("dyndep '%s' not in inputs", dyndep->s); + val = edgevar(e, "pool", true); if (val) e->pool = poolget(val->s); @@ -222,7 +233,8 @@ parsepool(struct scanner *s, struct environment *env) static void checkversion(const char *ver) { - if (strcmp(ver, ninjaversion) > 0) + int nmajor, nminor = 0; + if (sscanf(ver, "%d.%d", &nmajor, &nminor) < 1 || (nmajor > 1) || (nminor > 10)) fatal("ninja_required_version %s is newer than %s", ver, ninjaversion); } diff --git a/samu.c b/samu.c index fc97106..50fea86 100644 --- a/samu.c +++ b/samu.c @@ -8,6 +8,7 @@ #include "arg.h" #include "build.h" #include "deps.h" +#include "dyndep.h" #include "env.h" #include "graph.h" #include "log.h" @@ -230,6 +231,7 @@ main(int argc, char *argv[]) graphinit(); envinit(); parseinit(); + dyndepinit(); /* parse the manifest */ parse(manifest, rootenv); diff --git a/tool.c b/tool.c index 7554a9b..5981fce 100644 --- a/tool.c +++ b/tool.c @@ -8,6 +8,7 @@ #include #include "arg.h" #include "env.h" +#include "dyndep.h" #include "graph.h" #include "tool.h" #include "util.h" @@ -33,6 +34,9 @@ cleanedge(struct edge *e) int ret = 0; size_t i; + if (e->dyndep) + dyndepload(e->dyndep, true); + for (i = 0; i < e->nout; ++i) { if (cleanpath(e->out[i]->path) < 0) ret = -1;