cparser.lua

-- Copyright (c) Facebook, Inc. and its affiliates.
-- This source code is licensed under the MIT license found in the
-- LICENSE file in the root directory of this source tree.
--
-- Lua module to preprocess and parse C declarations.
-- (Leon Bottou, 2015)

-- @r-lyeh: del lcdecl module
-- @r-lyeh: add cppString method
-- @r-lyeh: add __COUNTER__ macro
-- @r-lyeh: del empty #line directives from output

-- standard libs
local string = require 'string'
local coroutine = require 'coroutine'
local table = require 'table'
local io = require 'io'

-- Lua 5.1 to 5.3 compatibility
local unpack = unpack or table.unpack

-- Debugging
local DEBUG = true
if DEBUG then pcall(require,'strict') end

-- luacheck: globals cparser
-- luacheck: ignore 43 4/ti 4/li
-- luacheck: ignore 212/.*_
-- luacheck: ignore 211/is[A-Z].* 211/Type
-- luacheck: ignore 542


---------------------------------------------------
---------------------------------------------------
---------------------------------------------------
-- ALL UGLY HACKS SHOULD BE HERE


-- Sometimes we cannot find system include files but need to know at
-- least things about them. For instance, certain system include files
-- define alternate forms for keywords.

local knownIncludeQuirks = {}

knownIncludeQuirks["<complex.h>"] = { -- c99
   "#ifndef complex", "# define complex _Complex", "#endif"
}

knownIncludeQuirks["<stdbool.h>"] = { -- c99
   "#ifndef bool", "# define bool _Bool", "#endif"
}

knownIncludeQuirks["<stdalign.h>"] = { -- c11
   "#ifndef alignof", "# define alignof _Alignof", "#endif",
   "#ifndef alignas", "# define alignas _Alignas", "#endif"
}

knownIncludeQuirks["<stdnoreturn.h>"] = { -- c11
   "#ifndef noreturn", "# define noreturn _Noreturn", "#endif"
}

knownIncludeQuirks["<threads.h>"] = { -- c11
   "#ifndef thread_local", "# define thread_local _Thread_local", "#endif"
}

knownIncludeQuirks["<iso646.h>"] = { -- c++
   "#define and &&", "#define and_eq &=", "#define bitand &", "#define bitor |",
   "#define compl ~", "#define not !", "#define not_eq !=", "#define or ||",
   "#define or_eq |=", "#define xor ^", "#define xor_eq ^="
}


---------------------------------------------------
---------------------------------------------------
---------------------------------------------------
-- TAGGED TABLES


-- Utilities to produce and print tagged tables.
-- The tag name is simply the contents of table key <tag>.
-- Function <newTag> returns a node constructor
--
-- Example:
--
-- > Foo = newTag('Foo')
-- > Bar = newTag('Bar')
--
-- > print( Foo{const=true,next=Bar{name="Hello"}} )
-- Foo{next=Bar{name="Hello"},const=true}
--
-- > print( Bar{name="hi!", Foo{1}, Foo{2}, Foo{3}} )
-- Bar{Foo{1},Foo{2},Foo{3},name="hi!"}

local function newTag(tag)
   -- the printing function
   local function tostr(self)
      local function str(x)
         if type(x)=='string' then
            return string.format("%q",x):gsub("\\\n","\\n")
         elseif type(x)=='table' and not getmetatable(x) then
            return "{..}"
         else
            return tostring(x)
         end
      end
      local p = string.format("%s{", self.tag or "Node")
      local s = {}
      local seqlen = 0
      for i=1,#self do
         if self[i] then seqlen=i else break end end
      for i=1,seqlen do
         s[1+#s] = str(self[i]) end
      for k,v in pairs(self) do
         if type(k) == 'number' then
            if k<1 or k>seqlen then
               s[1+#s] = string.format("[%s]=%s",k,str(v)) end
         elseif type(k) ~= 'string' then
            s.extra = true
         elseif k:find("^_") and type(v)=='table' then
            s[1+#s] = string.format("%s={..}",k) -- hidden
         elseif k ~= 'tag' then
            s[1+#s] = string.format("%s=%s",k,str(v)) end
      end
      if s.extra then s[1+#s] = "..." end
      return p .. table.concat(s,',') .. '}'
   end
   -- the constructor
   return function(t) -- must be followed by a table constructor
      t = t or {}
      assert(type(t)=='table')
      setmetatable(t, { __tostring=tostr } )
      t.tag = tag
      return t
   end
end

-- hack to print any table: print(Node(nn))
local Node = newTag(nil)   -- luacheck: ignore 211

---------------------------------------------------
---------------------------------------------------
---------------------------------------------------
-- UTILITIES


-- Many functions below have an optional argument 'options' which is
-- simply an array of compiler-like options that are specified in the
-- toplevel call and passed to nearly all functions. Because it
-- provides a good communication channel across the code components,
-- many named fields are also used for multiple purposes. The
-- following function is called at the beginning of the user facing
-- functions to make a copy of the user provided option array and
-- setup some of these fields.

local function copyOptions(options)
   options = options or {}
   assert(type(options)=='table')
   local noptions = {}
   -- copy options
   for k,v in ipairs(options) do noptions[k]=v end
   -- copy user modifiable named fields
   noptions.sizeof = options.sizeof     -- not used yet
   noptions.alignof = options.alignof   -- not used yet
   -- create reversed hash
   noptions.hash = {}
   for i,v in ipairs(options) do
      noptions.hash[v] = i
   end
   -- compute dialect flags
   local dialect = 'gnu99'
   for _,v in ipairs(options) do
      if v:find("^%-std=%s*[^%s]") then
         dialect = v:match("^%-std=%s*(.-)%s*$")
      end
   end
   noptions.dialect = dialect
   noptions.dialectGnu = dialect:find("^gnu")
   noptions.dialect99 = dialect:find("9[9x]$")
   noptions.dialect11 = dialect:find("1[1x]$")
   noptions.dialectAnsi = not noptions.dialectGnu
   noptions.dialectAnsi = noptions.dialectAnsi and not noptions.dialect99
   noptions.dialectAnsi = noptions.dialectAnsi and not noptions.dialect11
   -- return
   return noptions
end


-- This function tests whether a particular option has been given.

local function hasOption(options, opt)
   assert(options)
   assert(options.silent or options.hash)
   return options.hash and options.hash[opt]
end


-- Generic functions for error messages

local function xmessage(err, options, lineno, message, ...)
   local msg = string.format("cparser: (%s) ",lineno)
   msg = msg .. string.format(message,...)
   if options.silent then
      if err == 'error' then error(msg, 0) end
   else
      if err == 'warning' and hasOption(options, "-Werror") then err = 'error' end
      if err == 'error' or not hasOption(options, "-w") then print(msg) end
      if err == 'error' then error("cparser: aborted",0) end
   end
end

local function xwarning(options, lineno, message, ...)
   xmessage('warning', options, lineno, message, ...)
end

local function xerror(options, lineno, message, ...)
   xmessage('error', options, lineno, message, ...)
end

local function xassert(cond, ...)
   if not cond then xerror(...) end
end

local function xdebug(lineno,message,...)
   local msg = string.format("\t\t[%s] ", lineno)
   msg = msg .. string.format(message,...)
   print(msg)
end


-- Nil-safe max

local function max(a,b)
   a = a or b
   b = b or a
   return a > b and a or b
end


-- Deep table comparison
-- (not very efficient, no loop detection)

local function tableCompare(a,b)
   if a == b then
      return true
   elseif type(a) == 'table' and type(b) == 'table' then
      for k,v in pairs(a) do
         if not tableCompare(v,b[k]) then return false end
      end
      for k,v in pairs(b) do
         if not tableCompare(a[k],v) then return false end
      end
      return true
   else
      return false
   end
end


-- Concatenate two possibly null arrays

local function tableAppend(a1, a2)
   if not a1 then
      return a2
   elseif not a2 then
      return a1
   else
      local a = {}
      for _,v in ipairs(a1) do a[1+#a] = v end
      for _,v in ipairs(a2) do a[1+#a] = v end
      return a
   end
end

-- Concatenate strings from table (skipping non-string content.)

local function tableConcat(a)
   local b = {}
   for _,v in ipairs(a) do
      if type(v) == 'string' then b[1+#b]=v end end
   return table.concat(b)
end


-- Evaluate a lua expression, return nil on error.

local function evalLuaExpression(s)
   assert(type(s)=='string')
   local f = load(string.gmatch(s,".*"))
   local function r(status,...)
      if status then return ... end end
   return r(pcall(f or error))
end


-- Bitwise manipulations
-- try lua53 operators otherwise revert to iterative version

local bit = evalLuaExpression([[
   local bit = {}
   function bit.bnot(a) return ~a end
   function bit.bor(a,b) return a | b end
   function bit.band(a,b) return a & b end
   function bit.bxor(a,b) return a ~ b end
   function bit.lshift(a,b) return a < 0 and b < 0 and ~((~a) << b) or a << b end
   return bit
]])

if not bit then
   local function bor(a,b)
      local r, c, d = 0, 1, -1
      while a > 0 or b > 0 or a < -1 or b < -1 do
         if a % 2 > 0 or b % 2 > 0 then r = r + c end
         a, b, c, d = math.floor(a / 2), math.floor(b / 2), c * 2, d * 2 end
      if a < 0 or b < 0 then r = r + d end
      return r end
   bit = {}
   function bit.bnot(a) return -1-a end
   function bit.bor(a,b) return bor(a,b) end
   function bit.band(a,b) return -1-bor(-1-a,-1-b) end
   function bit.bxor(a,b) return bor(-1-bor(a,-1-b),-1-bor(-1-a,b)) end
   function bit.lshift(a,b) return math.floor(a * 2 ^ b) end
end


-- Coroutine helpers.
-- This code uses many coroutines that yield lines or tokens.
-- All functions that can yield take an options table as first argument.


-- Wrap a coroutine f into an iterator
-- The options and all the extra arguments are passed
-- to the coroutine when it starts. Together with the
-- above calling convention, this lets us specify
-- coroutine pipelines (see example in function "cpp".)

local function wrap(options, f, ...)
   local function g(...) coroutine.yield(nil) f(...) end
   local c = coroutine.create(g)
   coroutine.resume(c, options, ...)
   local function r(s,...)
      if not s then local m = ... ; error(m, 0) end
      return ...
   end
   return function()
      if coroutine.status(c) ~= 'dead' then
         return r(coroutine.resume(c))
      end
   end
end


-- Collect coroutine outputs into an array
-- The options and the extra arguments are passed to the coroutine.

local function callAndCollect(options, f, ...) -- Bell Labs nostalgia
   local collect = {}
   for s in wrap(options, f, ...) do
      collect[1+#collect] = s
   end
   return collect
end


-- Yields all outputs from iterator iter.
-- Argument options is ignored.

local function yieldFromIterator(options_, iter)
   local function yes(v,...) coroutine.yield(v,...) return v end
   while yes(iter()) do end
end


-- Yields all values from array <arr>.
-- This function successively yields all values in the table.
-- Every yield is augmented with all extra arguments passed to the function.
-- Argument options is ignored.

local function yieldFromArray(options_, arr, ...)
   for _,v in ipairs(arr) do
      coroutine.yield(v, ...)
   end
end





---------------------------------------------------
---------------------------------------------------
---------------------------------------------------
-- INITIAL PREPROCESSING



-- A routine that pulls lines from a line iterator
-- and yields them together with a location
-- composed of the optional prefix, a colon, and a line number.
-- Argument options is ignored.
-- Lua provides good line iterators such as:
--   io.lines(filename) filedesc:lines()  str:gmatch("[^\n]+")

local function yieldLines(options_,lineIterator,prefix)
   prefix = prefix or ""
   assert(type(prefix)=='string')
   local n = 0
   for s in lineIterator do
      n = n + 1
      coroutine.yield(s, string.format("%s:%d", prefix, n))
   end
end


-- A routine that obtains lines from coroutine <lines>,
-- joins lines terminated by a backslash, and yield the
-- resulting lines. The coroutine is initialized with
-- argument <options> and all extra arguments.
-- Reference: https://gcc.gnu.org/onlinedocs/cpp/Initial-processing.html (3)

local function joinLines(options, lines, ...)
   local li = wrap(options, lines, ...)
   for s, n in li do
      while type(s) == 'string' and s:find("\\%s*$") do
         local t = li() or ""
         s = s:gsub("\\%s*$", "") .. t
      end
      coroutine.yield(s, n)
   end
end


-- A routine that obtain lines from coroutine <lines>, eliminate the
-- comments and yields the resulting lines.  The coroutine is
-- initialized with argument <options> and all extra arguments.
-- Reference: https://gcc.gnu.org/onlinedocs/cpp/Initial-processing.html (4)

local function eliminateComments(options, lines, ...)
   local lineIterator = wrap(options, lines, ...)
   local s,n = lineIterator()
   while type(s) == 'string' do
      local inString = false
      local q = s:find("[\'\"\\/]", 1)
      while q ~= nil do
         if hasOption(options,"-d:comments") then
            xdebug(n, "comment: [%s][%s] %s",s:sub(1,q-1),s:sub(q),inString)
         end
         local c = s:byte(q)
         if inString then
            if c == 92 then -- \
               q = q + 1
            elseif c == inString then
               inString = false
            end
         else
            if c == 34 or c == 39 then -- " or '
               inString = c
            elseif c == 47 and s:byte(q+1) == 47 then -- "//"
               s = s:sub(1,q-1)
            elseif c == 47 and s:byte(q+1) == 42 then -- "/*"
               local p = s:find("%*/",q+2)
               if p ~= nil then
                  s = s:sub(1,q-1) .. " " .. s:sub(p+2)
               else
                  s = s:sub(1,q-1)
                  local ss,pp
                  repeat
                     ss = lineIterator()
                     xassert(ss ~= nil, options, n, "Unterminated comment")
                     pp = ss:find("%*/")
                  until pp
                  s = s .. " " .. ss:sub(pp+2)
               end
            end
         end
         q = s:find("[\'\"\\/]", q+1)
      end
      coroutine.yield(s, n)
      s, n = lineIterator()
   end
end



---------------------------------------------------
---------------------------------------------------
---------------------------------------------------
-- TOKENIZER



local keywordTable = {
   ------ Standard keywords
   "auto", "break", "case", "char", "const", "continue", "default", "do",
   "double", "else", "enum", "extern", "float", "for", "goto", "if", "int",
   "long", "register", "return", "short", "signed", "sizeof", "static", "struct",
   "switch", "typedef", "union", "unsigned", "void", "volatile", "while",
   ------ Nonstandard or dialect specific keywords do not belong here
   ------ because the main function of this table is to say which
   ------ identifiers cannot be variable names.
}

local punctuatorTable = {
   "+", "-", "*", "/", "%", "&", "|", "^", ">>", "<<", "~",
   "=", "+=", "-=", "*=", "/=", "%=", "&=", "|=", "^=", ">>=", "<<=",
   "(", ")", "[", "]", "{", "}", "++", "--",
   "==", "!=", ">=", "<=", ">", "<", "&&", "||", "!",
   ".", "->", "*", "&", "?", ":", "::", "->*", ".*", ";", ",",
   "#", "##", "...", "@", "\\" -- preprocessor stuff
}

local keywordHash = {}
for _,v in ipairs(keywordTable) do
   keywordHash[v] = true
end

local punctuatorHash = {}
for _,v in ipairs(punctuatorTable) do
   local l = v:len()
   local b = v:byte()
   punctuatorHash[v] = true
   punctuatorHash[b] = max(l,punctuatorHash[b])
end


-- The following functions test the types of the tokens returned by the tokenizer.
-- They should not be applied to arbitrary strings.

local function isSpace(tok)
   return type(tok) == 'string' and tok:find("^%s") ~= nil end
local function isNewline(tok) -- Subtype of space
   return type(tok) == 'string' and tok:find("^\n") ~= nil end
local function isNumber(tok)
   return type(tok) == 'string' and tok:find("^[.0-9]") ~= nil end
local function isString(tok)
   if type(tok) ~= 'string' then return false end
   return tok:find("^[\'\"]") ~= nil end
local function isHeaderName(tok)
   if type(tok) ~= 'string' then return false end
   return tok:find("^\"") or tok:find("^<") and tok:find(">$") end
local function isPunctuator(tok)
   return type(tok) == 'string' and punctuatorHash[tok] ~= nil end
local function isIdentifier(tok)
   return type(tok) == 'string' and tok:find("^[A-Za-z_$]") ~= nil end
local function isKeyword(tok) -- Subtype of identifier
   return keywordHash[tok] ~= nil end
local function isName(tok) -- Subtype of identifier
   return isIdentifier(tok) and not keywordHash[tok] end

-- Magic tokens are used to mark macro expansion boundaries (see expandMacros.)
local function isMagic(tok)
   return tok and type(tok) ~= 'string' end
local function isBlank(tok) -- Treats magic token as space.
   return isMagic(tok) or isSpace(tok) end

-- The tokenizeLine() function takes a line, splits it into tokens,
-- and yields tokens and locations. The number tokens are the weird
-- preprocessor numbers defined by ansi c. The string tokens include
-- character constants and angle-bracket delimited strings occuring
-- after an include directive. Every line begins with a newline
-- token giving the proper indentation. All subsequent spaces
-- are reduced to a single space character.

local function tokenizeLine(options, s, n, notNewline)
   -- little optimization for multiline macros
   -- s may be an array of precomputed tokens
   if type(s) == 'table' then
      return yieldFromArray(options, s, n)
   end
   -- normal operation
   assert(type(s) == 'string')
   local p = s:find("[^%s]")
   -- produce a newline token
   if p and not notNewline then
      local r = '\n' .. s:sub(1,p-1)
      coroutine.yield(r, n)
   end
   -- produce one token
   local function token()
      local b, l, r
      if hasOption(options, "-d:tokenize") then
         xdebug(n, "[%s][%s]",s:sub(1,p-1),s:sub(p))
      end
      -- space
      l = s:find("[^%s]", p)
      if l == nil then
         return nil
      elseif l > p then
         p = l
         return " ", n
      end
      -- identifier
      r = s:match("^[a-zA-Z_$][a-zA-Z0-9_$]*", p)
      if r ~= nil then
         p = p + r:len()
         return r, n
      end
      -- preprocessor numbers
      r = s:match("^%.?[0-9][0-9a-zA-Z._]*", p)
      if r ~= nil then
         l = r:len()
         while r:find("[eEpP]$") and s:find("^[-+]", p+l) do
            r = r .. s:match("^[-+][0-9a-zA-Z._]*", p+l)
            l = r:len()
         end
         p = p + l
         return r, n
      end
      -- angle-delimited strings in include directives
      b = s:byte(p)
      if b == 60 and s:find("^%s*#%s*include") then
         r = s:match("^<[^>]+>", p)
         if r ~= nil then
            p = p + r:len()
            return r, n
         end
      end
      -- punctuator
      l = punctuatorHash[b]
      if l ~= nil then
         while l > 0 do
            r = s:sub(p,p+l-1)
            if punctuatorHash[r] then
               p = p + l
               return r, n
            end
            l = l - 1
         end
      end
      -- string
      if b == 34 or b == 39 then -- quotes
         local q = p
         repeat
            q = s:find("[\'\"\\]", q + 1)
            l = s:byte(q)
            xassert(q ~= nil, options, n, "Unterminated string or character constant")
            if l == 92 then
               q = q + 1
            end
         until l == b
         r = s:sub(p,q)
         p = q + 1
         return r, n
      end
      -- other stuff (we prefer to signal an error here)
      xerror(options, n,"Unrecognized character (%s)", s:sub(p))
   end
   -- loop
   if p then
      for tok,tokn in token do
         coroutine.yield(tok, tokn)
      end
   end
end


-- Obtain lines from coroutine <lines>,
-- and yields their tokens. The coroutine is initialized with
-- argument <options> and all extra arguments.

local function tokenize(options, lines, ...)
   for s,n in wrap(options, lines, ...) do
      tokenizeLine(options, s, n)
   end
end


---------------------------------------------------
---------------------------------------------------
---------------------------------------------------
-- PREPROCESSING


-- Preprocessing is performed by two coroutines. The first one
-- processes all the preprocessor directives and yields the remaining
-- lines. The second one processes tokens from the remaining lines and
-- perform macro expansions. Both take a table of macro definitions as
-- argument. The first one writes into the table and the second one
-- reads from it.
--
-- Each macro definition is an array of tokens (for a single line
-- macro) or a table whose entry <"lines"> contains an array of arrays
-- of tokens (#defmacro). If the macro takes arguments, the entry
-- <"args"> contains a list of argument names. If the macro is
-- recursive (#defrecmacro), the entry <recursive> is set.
-- Alternatively, the macro definition may be a function called at
-- macro-expansion time. This provides for complicated situations.


-- forward declarations
local function expandMacros() end
local function processDirectives() end


-- Starting with the second coroutine which takes a token producing
-- coroutine and yields the preprocessed tokens. Argument macros is
-- the macro definition table.

-- The standard mandates that the result of a macro-expansion must be
-- scanned for further macro invocations whose argunent list possibly
-- consume tokens that follow the macro-expansion. This means that one
-- cannot recursively call expandMacros but one must prepend the
-- macro-expansion in front of the remaining tokens. The standard also
-- mandates that the result of any macro-expansion must be marked to
-- prevent recursive invocation of the macro that generated it,
-- whether when expanding macro arguments or expanding the macro
-- itself.  We achieve this by bracketing every macro-expansion with
-- magic tokens that track which macro definitions must be disabled.
-- These magic tokens are removed later in the coroutines
-- <filterSpaces> or <preprocessedLines>.

expandMacros = function(options, macros, tokens, ...)
   -- basic iterator
   local ti = wrap(options, tokens, ...)
   -- prepending tokens in front of the token stream
   local prepend = {}
   local function prependToken(s,n)
      table.insert(prepend,{s,n}) end
   local function prependTokens(pti)
      local pos = 1+#prepend
      for s,n in pti do table.insert(prepend,pos,{s,n}) end end
   local ti = function()
      if #prepend > 0 then return unpack(table.remove(prepend))
      else return ti() end end
   -- iterator that handles magic tokens to update macro definition table
   local ti = function()
      local s,n = ti()
      while type(s) == 'table' do
         if s.tag == 'push' then
            local nmacros = {}
            setmetatable(nmacros, {__index=macros})
            if s.symb then nmacros[s.symb] = false end
            macros = nmacros
         elseif s.tag == 'pop' then
            local mt = getmetatable(macros)
            if mt and mt.__index then macros = mt.__index end
         end
         coroutine.yield(s,n)
         s,n = ti()
      end
      return s,n
   end
   -- redefine ti() to ensure tok,n remain up-to-date
   local tok,n = ti()
   local ti = function() tok,n=ti() return tok,n end
   -- collect one macro arguments into an array
   -- stop when reaching a closing parenthesis or a comma
   local function collectArgument(ti, varargs)
      local count = 0
      local tokens = {}
      ti()
      while isSpace(tok) do
         tok = ti()
      end
      while tok do
         if tok == ')' and count == 0 then
            break
         elseif tok == ')' then
            count = count - 1
         elseif tok == '(' then
            count = count + 1
         elseif tok == ',' and count == 0 and not varargs then
            break
         end
         if isSpace(tok) then tok = " " end
         tokens[1+#tokens] = tok
         tok = ti()
      end
      if #tokens > 0 and isSpace(tokens[#tokens]) then
         tokens[#tokens] = nil
      end
      return tokens
   end
   -- collects all macro arguments
   local function collectArguments(ti,def,ntok,nn)
      local args = def.args
      local nargs = { [0]={} }
      if #args == 0 then ti() end
      for _,name in ipairs(args) do
         if tok == ')' and name == "__VA_ARGS__" then
            nargs[0][name] = { negComma=true }
            nargs[name] = { negComma=true }
         else
            xassert(tok=='(' or tok==',', options, nn, "not enough arguments for macro '%s'", ntok)
            local arg = collectArgument(ti, name == "__VA_ARGS__")
            nargs[0][name] = arg
            nargs[name] = callAndCollect(options, expandMacros, macros, yieldFromArray, arg, nn)
         end
      end
      if def.nva then -- named variadic argument (implies dialectGnu)
         nargs[def.nva] = nargs["__VA_ARGS__"]
         nargs[0][def.nva] = nargs[0]["__VA_ARGS__"]
      end
      xassert(tok, options, nn, "unterminated arguments for macro '%s'", ntok)
      xassert(tok==')', options, nn, "too many arguments for macro '%s'", ntok)
      return nargs
   end
   -- coroutine that substitute the macro arguments
   -- and stringification and concatenation are handled here
   local function substituteArguments(options, def, nargs, n, inDirective)
      local uargs = nargs[0] or nargs        -- unexpanded argument values
      if inDirective then nargs = uargs end  -- use unexpanded arguments in directives
      -- prepare loop
      local i,j,k = 1,1,1
      while def[i] do
         if isBlank(def[i]) then
            -- copy blanks
            coroutine.yield(def[i], n)
         else
            -- positions j and k on next non-space tokens
            local function updateJandK()
               if j <= i then j=i
                  repeat j=j+1 until def[j] == nil or not isBlank(def[j]) end
               if k <= j then k=j
                  repeat k=k+1 until def[k] == nil or not isBlank(def[k]) end
            end
            updateJandK()
            -- alternatives
            if def[i]=='#' and def[j] and nargs[def[j]] then
               -- stringification (with the weird quoting rules)
               local v = { '\"' }
               for _,t in ipairs(uargs[def[j]]) do
                  if type(t)=='string' then
                     if t:find("^%s+$") then t = ' ' end
                     if t:find("^[\'\"]") then t = string.format("%q", t):sub(2,-2) end
                     v[1+#v] = t end end
               v[1+#v] = '\"'
               coroutine.yield(tableConcat(v), n)
               i = j
            elseif def.nva and def[i]==',' and def[j]=='##' and def[k]==def.nva then
               -- named variadic macro argument with ## to signal negative comma (gcc crap)
               if nargs[def.nva].negComma then i=i+1 end
               while i < j do coroutine.yield(def[i], n) ; i=i+1 end
            elseif def[i]==',' and def[j]=='__VA_ARGS__' and def[k]==')' then
               -- __VA_ARGS__ with implied negative comma semantics
               if nargs[def[j]].negComma then i=i+1 end
               while i < j do coroutine.yield(def[i], n) ; i=i+1 end
               i = j-1
            elseif def[j]=='##' and def[k] and not inDirective then
               -- concatenation
               local u = {}
               local function addToU(s)
                  if nargs[s] then for _,v in ipairs(uargs[s]) do u[1+#u] = v end
                  else u[1+#u]=s end end
               addToU(def[i])
               while def[j] == '##' and def[k] do
                  addToU(def[k])
                  i = k
                  updateJandK()
               end
               tokenizeLine(options, tableConcat(u), n, true)
            elseif nargs[def[i]] then
               -- substitution
               yieldFromArray(options, nargs[def[i]], n)
            else
               -- copy
               coroutine.yield(def[i], n)
            end
         end
         i = i + 1
      end
   end
   -- main loop
   local newline, directive = true, false
   while tok ~= nil do
      -- detects Zpassed directives
      if newline and tok == '#' then
         newline, directive = false, true
      elseif not isBlank(tok) then
         newline = false
      elseif isNewline(tok) then
         newline, directive = true, false
      end
      -- process code
      local def = macros[tok]
      if not def or directive then
         -- not a macro
         coroutine.yield(tok, n)
      elseif type(def) == 'function' then
         -- magic macro
         def(ti,tok,n)
      elseif def.args == nil then
         -- object-like macro
         prependToken({tag='pop'},n)
         prependTokens(wrap(options, substituteArguments, def, {}, n))
         prependToken({tag='push', symb=tok},n)
      else
         -- function-like macro
         local ntok, nn = tok,n
         local spc = false
         ti()
         if isSpace(tok) then spc=true ti() end
         if (tok ~= '(') then
            coroutine.yield(ntok, nn)
            if spc then coroutine.yield(' ', n) end
            if tok then prependToken(tok,n) end
         else
            local nargs = collectArguments(ti,def,ntok,nn)
            if def.lines == nil then
               -- single-line function-like macro
               prependToken({tag='pop'},n)
               prependTokens(wrap(options, substituteArguments, def, nargs, nn))
               prependToken({tag='push', symb=ntok},nn)
            else
               -- multi-line function-like macro
               local lines = def.lines
               -- a coroutine that yields the macro definition
               local function yieldMacroLines()
                  local count = 0
                  for i=1,#lines,2 do
                     local ls,ln = lines[i], lines[i+1]
                     -- are we possibly in a cpp directive
                     local dir = false
                     if ls[2] and ls[2]:find('^#') then
                        dir = isIdentifier(ls[3]) and ls[3] or ls[4]
                     end
                     if dir and nargs[dir] then
                        dir = false      -- leading stringification
                     elseif dir == 'defmacro' then
                        count = count + 1  -- entering a multiline macto
                     elseif dir == 'endmacro' then
                        count = count - 1  -- leaving a multiline macro
                     end
                     dir = dir or count > 0
                     -- substitute
                     ls = callAndCollect(options,substituteArguments,ls,nargs,ln,dir)
                     -- compute lines (optimize speed by passing body lines as tokens)
                     local j=1
                     while isBlank(ls[j]) do j=j+1 end
                     if ls[j] and ls[j]:find("^#") then -- but not directives
                        ls = ls[1]:sub(2) .. tableConcat(ls, nil, 2)
                     end
                     coroutine.yield(ls,ln)
                  end
               end
               -- recursively reenters preprocessing subroutines in order to handle
               -- preprocessor directives located inside the macro expansion. As a result
               -- we cannot expand macro invocations that extend beyond the macro-expansion.
               local nmacros = {}
               setmetatable(nmacros,{__index=macros})
               if not def.recursive then nmacros[ntok]=false end
               if not def.recursive then coroutine.yield({tag='push',symb=ntok}) end
               expandMacros(options, nmacros, tokenize, processDirectives, nmacros, yieldMacroLines)
               if not def.recursive then coroutine.yield({tag='pop'}) end
            end
         end
      end
      ti()
   end
end


-- Processing conditional directive requires evaluating conditions
-- This function takes an iterator on preprocessed expression tokens
-- and computes the value. This does not handle defined(X) expressions.
-- Optional argument resolver is a function that takes an indentifer
-- name and returns a value. Otherwise zero is assumed

local function evaluateCppExpression(options, tokenIterator, n, resolver)
   -- redefine token iterator to skip spaces and update tok
   local tok
   local function ti()
      repeat tok = tokenIterator()
      until not isBlank(tok) return tok
   end
   -- operator tables
   local unaryOps = {
      ["!"] = function(v) return v == 0 and 1 or 0 end,
      ["~"] = function(v) return bit.bnot(v) end,
      ["+"] = function(v) return v end,
      ["-"] = function(v) return -v end,
      ["L"] = function(v) return v end
   }
   local binaryOps = {
      ["*"] = function(a,b) return a * b end,
      ["/"] = function(a,b) xassert(b~=0,options,n,"division by zero"); return math.floor(a / b) end,
      ["%"] = function(a,b) xassert(b~=0,options,n,"division by zero"); return a % b end,
      ["+"] = function(a,b) return a + b end,
      ["-"] = function(a,b) return a - b end,
      [">>"] = function(a,b) return bit.lshift(a, -b) end,
      ["<<"] = function(a,b) return bit.lshift(a, b) end,
      [">="] = function(a,b) return a >= b and 1 or 0 end,
      ["<="] = function(a,b) return a <= b and 1 or 0 end,
      [">"] = function(a,b) return a > b and 1 or 0 end,
      ["<"] = function(a,b) return a < b and 1 or 0 end,
      ["=="] = function(a,b) return a == b and 1 or 0 end,
      ["!="] = function(a,b) return a ~= b and 1 or 0 end,
      ["&"] = function(a,b) return bit.band(a,b) end,
      ["^"] = function(a,b) return bit.bxor(a,b) end,
      ["|"] = function(a,b) return bit.bor(a,b) end,
      ["&&"] = function(a,b) return (a ~= 0 and b ~= 0) and 1 or 0 end,
      ["||"] = function(a,b) return (a ~= 0 or b ~= 0) and 1 or 0 end,
   }
   local binaryPrec = {
      ["*"] = 1, ["/"] = 1, ["%"] = 1,
      ["+"] = 2, ["-"] = 2,
      [">>"] = 3, ["<<"] = 3,
      [">="] = 4, ["<="] = 4, ["<"] = 4, [">"] = 4,
      ["=="] = 5, ["!="] = 5,
      ["&"] = 6, ["^"] = 7, ["|"] = 8,
      ["&&"] = 9, ["||"] = 10
   }
   -- forward
   local function evaluate() end
   -- unary operations
   local function evalUnary()
      if unaryOps[tok] then
         local op = unaryOps[tok]
         ti(); return op(evalUnary())
      elseif tok == '(' then
         ti(); local v = evaluate()
         xassert(tok == ')', options, n, "missing closing parenthesis")
         ti(); return v
      elseif tok == 'defined' then -- magic macro should have removed this
         xerror(options, n, "syntax error after <defined>")
      elseif isIdentifier(tok) then
         local v = type(resolver) == 'function' and resolver(tok,ti)
         ti(); return v or 0
      elseif isNumber(tok) then
         local v = tok:gsub("[ULul]+$","")
         if v:find("^0[0-7]+$") then
            v = tonumber(v,8) -- octal
         elseif v:find("^0[bB][01]+") then
            v = tonumber(v:sub(3),2) -- binary
         else
            v = tonumber(v) -- lua does the rest
         end
         xassert(v and v == math.floor(v), options, n, "syntax error (invalid integer '%s')", tok)
         ti(); return v
      elseif isString(tok) then
         local v = '""'
         if tok:find("^'") then -- interpret character constant as number
            v = evalLuaExpression(string.format("return string.byte(%s)", tok))
            xassert(type(v)=='number', options, n, "syntax error (invalid value '%s')", tok)
            ti()
         else
            while isString(tok) do
               xassert(tok:find('^"'), options, n, "syntax error (invalid value '%s')", tok)
               v = v:gsub('"$','') .. tok:gsub('^"','')
               ti()
            end
         end
         return v
      end
      xerror(options, n, "syntax error (invalid value '%s')", tok)
   end
   -- binary operations
   local function evalBinary(p)
      if p == 0 then
         return evalUnary()
      else
         local val = evalBinary(p-1)
         while binaryPrec[tok] == p do
            local op = binaryOps[tok]  ti()
            local oval = evalBinary(p-1)
            xassert(p==4 or p==5 or type(val)=='number', options, n,
                    "expression uses arithmetic operators on strings")
            xassert(type(val) == type(oval), options, n,
                    "expression compares numbers and strings")
            val = op(val,oval)
         end
         return val
      end
   end
   -- eval ternary conditonal
   local function evalTernary()
      local c = evalBinary(10)
      if tok ~= '?' then return c end
      ti() local v1 = evalBinary(10)
      xassert(tok == ':', options, n, "expecting ':' after '?'")
      ti() local v2 = evalBinary(10)
      if c == 0 then return v2 else return v1 end
   end

   -- actual definition of evaluate
   evaluate = function()
      return evalTernary()
   end
   -- main function
   ti()
   xassert(tok, options, n, "constant expression expected");
   local result = evaluate()
   if hasOption(options, "-d:eval") then
      xdebug(n, "eval %s", result)
   end
   -- warn about garbage when called from cpp (but not when called with resolver)
   while isBlank(tok) do ti() end
   xassert(resolver or not tok, options, n, "garbage after conditional expression");
   return result
end


-- Now dealing with the coroutine that processes all directives.
-- This coroutine obtains lines from coroutine <lines>,
-- processes all directives, and yields remaining lines

processDirectives = function(options, macros, lines, ...)
   local li = wrap(options, lines, ...)
   local s, n = li()
   -- redefine li to make sure vars s and n are up-to-date
   local li = function() s,n=li() return s,n end
   -- directives store their current token in these vars
   local dirtok, tok, spc
   -- forward declaration
   local processLine
   -- the captureTable mechanism communicates certain preprocessor
   -- events to the declaration parser in order to report them
   -- to the user (the parsing does not depend on this).
   -- if macros[1] is a table, the preprocessor will append
   -- records to this table for the parser to process.
   local function hasCaptureTable()
      local captable = rawget(macros,1)
      return captable and type(captable)=='table' and captable
   end
   local function addToCaptureTable(record)
      local captable = hasCaptureTable()
      if captable and record then captable[1+#captable] = record end
   end

   -- simple directives
   local function doIgnore()
      if hasOption(options, "-Zpass") then coroutine.yield(s, n) end
   end
   local function doError()
      xerror(options, n, "unexpected preprocessor directive #%s", dirtok)
   end
   local function doMessage()
      local msg = s:match("^%s*#+%s*[a-z]*%s+([^%s].*)")
      xmessage(dirtok, options, n, msg or '#' .. dirtok)
   end
   -- undef
   local function doUndef(ti)
      ti()
      local nam = tok
      xassert(isIdentifier(nam), options, n, "symbol expected after #undef")
      if ti() then xwarning(options, n, "garbage after #undef directive") end
      if hasOption(options, "-d:defines") then xdebug(n, "undef %s", nam) end
      if hasCaptureTable() and macros[nam] and macros[nam].captured then
         addToCaptureTable{directive='undef',name=nam, where=n}
      end
      macros[nam] = false -- false overrides inherited definitions
   end
   -- define
   local function getMacroArguments(ti)
      local args = {}
      local msg = "argument list in function-like macro"
      local nva = nil -- named variadic argument
      ti()
      while tok and tok ~= ')' do
         local nam = tok
         ti()
         if options.dialectGnu and isIdentifier(nam) and tok == '...' then nam,nva=tok,nam ; ti() end
         xassert(nam ~= "__VA_ARGS__", options, n, "name __VA_ARGS__ is not allowed here")
         xassert(tok == ')' or nam ~= '...', options, n, "ellipsis in argument list must appear last")
         xassert(tok == ')' or tok == ',', options, n, "bad " .. msg)
         if tok == ',' then ti() end
         if nam == '...' then nam = "__VA_ARGS__" end
         xassert(isIdentifier(nam), options, n, "bad " .. msg)
         args[1+#args] = nam
      end
      xassert(tok == ')', options, n, "unterminated " .. msg)
      ti()
      return args, nva
   end
   local function doDefine(ti)
      xassert(isIdentifier(ti()), options, n, "symbol expected after #define")
      local nam, args, nva = tok, nil, nil
      -- collect arguments
      if ti() == '(' and not spc then
         args, nva = getMacroArguments(ti)
      end
      -- collect definition
      local def = { tok, args = args, nva = nva }
      while ti(true) do
         def[1+#def] = tok
      end
      -- define macro
      if macros[nam] and not tableCompare(def,macros[nam]) then
         xwarning(options, n,"redefinition of preprocessor symbol '%s'", nam)
      end
      macros[nam] = def
      -- debug
      if hasOption(options, "-d:defines") then
         if args then args = "(" .. tableConcat(args,",") .. ")"  end
         xdebug(n, "define %s%s = %s", nam, args or "", tableConcat(def,' '))
      end
      -- capture integer macro definitions
      if hasCaptureTable() and args == nil then
         local i = 0
         local v = callAndCollect(options, expandMacros, macros, yieldFromArray, def, n)
         local function ti() i=i+1 return v[i] end
         local ss,r = pcall(evaluateCppExpression,{silent=true}, ti, n, error)
         if ss and type(r)=='number' then
            def.captured = true
            addToCaptureTable{directive='define', name=nam, intval=r, where=n}
         end
      end
   end
   -- defmacro
   local function checkDirective(stop)
      xassert(s, options, n, "unterminated macro (missing #%s)", stop)
      local r = type(s) == 'string' and s:match("^%s*#%s*([a-z]+)")
      if r == "endmacro" or r == "endif" then
         if s:find(r .. "%s*[^%s]") then
            xwarning(options, n, "garbage after #%s directive", r)
         end
      end
      return r
   end
   local function doMacroLines(lines, stop)
      while true do
         li()
         local ss = callAndCollect(options,tokenizeLine,s,n)
         if #ss > 0 then lines[1+#lines] = ss ; lines[1+#lines] = n end
         local r = checkDirective(stop)
         if r == "endmacro" or r == "endif" then
            xassert(r == stop, options,n, "unbalanced directives (got #%s instead of #%s)",r,stop)
            return r
         elseif r=="defmacro" then
            doMacroLines(lines,"endmacro")
         elseif r == "if" or r == "ifdef" or r == "ifndef" then
            doMacroLines(lines,"endif")
         end
      end
   end
   local function doDefmacro(ti)
      xassert(isIdentifier(ti()), options, n, "symbol expected after #defmacro")
      local nam,nn = tok,n
      xassert(ti()=='(', options, n, "argument list expected in #defmacro")
      local args, nva = getMacroArguments(ti)
      xassert(not tok, options, n, "garbage after argument list in #defmacro")
      -- collect definition
      local lines = {}
      local def = { args=args, nva=nva, lines=lines, recursive=(dirtok=="defrecursivemacro") }
      doMacroLines(lines,"endmacro")
      lines[#lines] = nil
      lines[#lines] = nil
      if hasOption(options,"-d:directives") then
         xdebug(n, "directive: #endmacro")
      end
      if macros[nam] and not tableCompare(def,macros[nam]) then
         xwarning(options, n, "redefinition of preprocessor symbol '%s'", nam)
      end
      if hasOption(options, "-d:defines") then
         xdebug(nn, "defmacro %s(%s) =", nam, tableConcat(args,','))
         for i=1,#lines,2 do
            xdebug(lines[i+1], "\t%s", tableConcat(lines[i]):gsub("^\n","")) end
      end
      macros[nam] = def
   end
   -- include
   local function doInclude(ti)
      -- get filename
      local pti = wrap(options, expandMacros, macros, yieldFromIterator, ti)
      local tok = pti()
      while isBlank(tok) do tok=pti() end
      if tok == '<' then              -- computed include
         repeat local tok2 = pti()
           tok = tok .. tostring(tok2)
         until tok2==nil or tok2=='>' or isNewline(tok2)
         tok = tok:gsub('%s>$','>')   -- gcc does this
      end
      xassert(isHeaderName(tok), options, n, "malformed header name after #include")
      local ttok = pti()
      while isBlank(ttok) do ttok=pti() end
      if ttok then xwarning(options, n, "garbage after #include directive") end
      -- interpret filename
      local sys = tok:byte() == 60
      local min = dirtok=="include_next" and options.includedir or 0
      local fname = evalLuaExpression(string.format("return '%s'", tok:sub(2,-2)))
      local pname, fd, fdi
      for i,v in ipairs(options) do
         if v == "-I-" then
            sys=false
         elseif i > min and v:find("^%-I") and not sys then
            pname = v:match("^%-I%s*(.*)") .. '/' .. fname
            fdi, fd = i, io.open(pname, "r")
            if fd then break end
         end
      end
      if fd then
         -- include file
         if hasOption(options, "-d:include") then xdebug(n, "including %q", pname) end
         local savedfdi = options.includedir
         options.includedir = fdi -- saved index to implement include_next
         processDirectives(options, macros, eliminateComments, joinLines,
                           yieldLines, fd:lines(), pname)
         options.includedir = savedfdi
      else
         -- include file not found
         if hasOption(options, "-Zpass") then
            coroutine.yield(string.format("#include %s",tok), n)
         else
            xwarning(options, n, "include directive (%s) was unresolved", tok)
         end
         -- quirks
         if knownIncludeQuirks[tok] then
            processDirectives(options, macros, eliminateComments, joinLines,
                              yieldFromArray, knownIncludeQuirks[tok], n)
         end
         -- capture
         if hasCaptureTable() then
            addToCaptureTable{directive='include',name=tok, where=n}
         end
      end
   end
   -- conditionals
   local function doConditionalBranch(execute)
      checkDirective("endif")
      while true do
         li()
         local r = checkDirective("endif")
         if r == "else" or r == "elif" or r == "endif" then
            return r
         elseif execute then
            processLine()
         elseif r == "if" or r == "ifdef" or r == "ifndef" then
            while doConditionalBranch(false) ~= "endif" do end
         end
      end
   end
   local function doConditional(result)
      local r = doConditionalBranch(result)
      if r == 'elif' and not result then
         return processLine(true)
      end
      while r ~= "endif" do
         r = doConditionalBranch(not result)
      end
      if hasOption(options,"-d:directives") then
         xdebug(n, "directive: %s",s:gsub('^%s*',''))
      end
   end
   local function doIfdef(ti)
      ti()
      xassert(isIdentifier(tok), options, n, "symbol expected after #%s", dirtok)
      local result = macros[tok]
      if ti() then xwarning(options, n, "garbage after #undef directive") end
      if dirtok == 'ifndef' then result = not result end
      doConditional(result)
   end
   local function doIf(ti)
      -- magic macro for 'defined'
      local nmacros = {}
      setmetatable(nmacros,{__index=macros})
      nmacros['defined'] = function(ti)
         local tok,n = ti()
         if tok == '(' then tok = ti()
            if ti() ~= ')' then tok = nil end end
         if isIdentifier(tok) then
            coroutine.yield(macros[tok] and "1" or "0", n)
         else
            coroutine.yield("defined", n) -- error
         end
      end
      -- evaluate and branch
      local pti = wrap(options, expandMacros, nmacros, yieldFromIterator, ti)
      local result = evaluateCppExpression(options, pti, n)
      doConditional(result ~= 0)
   end
   -- table of directives
   local directives = {
      ["else"] = doError, ["elif"] = doError, ["endif"] = doError,
      ["pragma"] = doIgnore, ["ident"] = doIgnore, ["line"] = doIgnore,
      ["error"] = doMessage, ["warning"] = doMessage,
      ["if"] = doIf, ["ifdef"] = doIfdef, ["ifndef"] = doIfdef,
      ["define"] = doDefine, ["undef"] = doUndef,
      ["defmacro"] = doDefmacro, ["defrecursivemacro"] = doDefmacro,
      ["endmacro"] = doError,
      ["include"] = doInclude, ["include_next"] = doInclude,
   }
   -- process current line
   processLine = function(okElif)
      if type(s) == 'table' then
         -- optimization for multiline macros:
         -- When s is an an array of precomputed tokens, code is assumed.
         coroutine.yield(s, n)
      elseif not s:find("^%s*#") then
         -- code
         coroutine.yield(s, n)
      elseif s:find("^%s*##") and hasOption(options, "-Zpass") then
         -- pass
         local ns = s:gsub("^(%s*)##","%1#")
         coroutine.yield(ns, n)
      else
         if hasOption(options, "-d:directives") then
            xdebug(n, "directive: %s",s:gsub('^%s*',''))
         end
         -- tokenize directive
         local ti = wrap(options, tokenizeLine, s, n)
         -- a token iterator that skips spaces unless told otherwise
         local ti = function(keepSpaces)
            tok = ti()
            spc = isSpace(tok)
            while not keepSpaces and isBlank(tok) do
               tok = ti()
               spc = spc or isSpace(tok)
            end
            return tok, n
         end
         -- start parsing directives
         ti()
         assert(tok=='#' or tok=='##')
         if tok == '##' then
            xwarning(options, n, "directive starts with ## without -Zpass") end
         dirtok = ti()
         if isIdentifier(tok) then
            local f = directives[dirtok]
            if okElif and dirtok == "elif" then f = doIf end
            xassert(f, options, n, "unrecognized preprocessor directive #%s", tok)
            f(ti)
         elseif tok ~= nil then
            xerror(options, n, "unrecognized preprocessor directive '#%s'", s:gsub("^%s*",""))
         end
      end
   end
   -- main loop
   while s ~= nil do
      processLine()
      li()
   end
end


-- This function yields initialization lines

local function initialDefines(options)
   -- cpp-extracted definitions
   if hasOption(options, "-Zcppdef") then
      local fd = io.popen("cpp -dM < /dev/null","r")
      yieldLines(options, fd:lines(), "<cppdef>")
      fd:close()
   end
   -- builtin definitions
   local sb = { "#define __CPARSER__ 1" }
   local function addDef(s,v)
      sb[1+#sb] = string.format("#ifndef %s",s)
      sb[1+#sb] = string.format("# define %s %s",s,v)
      sb[1+#sb] = string.format("#endif")
   end
   addDef("__STDC__", "1")
   local stdc = "199409L"
   if options.dialect11 then stdc = "201112L" end
   if options.dialect99 then stdc = "199901L" end
   addDef("__STDC_VERSION__", stdc)
   if options.dialectGnu then
      addDef("__GNUC__", 4)
      addDef("__GNUC_MINOR__", 2)
   end
   yieldLines(options, wrap(options, yieldFromArray, sb), "<builtin>")
   -- command line definitions
   local sc = {}
   for _,v in ipairs(options) do
      local d
      if v:find("^%-D(.*)=") then
         d = v:gsub("^%-D%s*(.*)%s*=%s*(.-)%s*$","#define %1 %2")
      elseif v:find("^%-D") then
         d = v:gsub("^%-D%s*(.-)%s*$","#define %1 1")
      elseif v:find("^%-U") then
         d = v:gsub("^%-U%s*(.-)%s*$","#undef %1")
      end
      if d then sc[1+#sc] = d end
   end
   yieldLines(options, wrap(options, yieldFromArray, sc), "<cmdline>")
end


-- This function creates the initial macro directory

local function initialMacros(options)
   local macros = {}
   -- magic macros
   macros["__FILE__"] = function(_,_,n)
      local f
      if type(n) == 'string' then f=n:match("^[^:]*") end
      coroutine.yield(string.format("%q", f or "<unknown>"), n)
   end
   macros["__LINE__"] = function(_,_,n)
      local d = n
      if type(d) == 'string' then d=tonumber(d:match("%d*$")) end
      coroutine.yield(string.format("%d", d or 0), n)
   end
   macros["__DATE__"] = function(_,_,n)
      coroutine.yield(string.format("%q", os.date("%b %e %Y")), n)
   end
   macros["__TIME__"] = function(_,_,n)
      coroutine.yield(string.format("%q", os.date("%T")), n)
   end
   --< @r-lyeh: add __COUNTER__
   macros["__COUNTER__"] = function(_,_,n)
      counter = counter or 0
      counter = counter + 1
      coroutine.yield(string.format("%d", counter), n)
   end
   --<
   -- initial macros
   local li = wrap(options,processDirectives,macros,initialDefines)
   for _ in li do end
   -- return
   return macros
end


-- This function prepares a string containing the definition of the
-- macro named <name> in macro definition table <macros>, or nil if no
-- such definition exists.

local function macroToString(macros, name)
   local v = macros[name]
   if type(v) == 'table' then
      local dir = 'define'
      if v.recursive and v.lines then
         dir = 'defrecursivemacro'
      elseif v.lines then
         dir = 'defmacro'
      end
      local arr = {"#", dir, ' ', name }
      if v.args then
         arr[1+#arr] = '('
         for i,s in ipairs(v.args) do
            if i ~= 1 then arr[1+#arr] = ',' end
            if s == '__VA_ARGS__' then s = (v.nva or '') .. '...' end
            arr[1+#arr] = s
         end
         arr[1+#arr] = ') '
      else
         arr[1+#arr] = ' '
      end
      for _,s in ipairs(v) do
         arr[1+#arr] = s
      end
      if v.lines then
         for i = 1, #v.lines, 2 do
            local vl = v.lines[i]
            arr[1+#arr] = '\n'
            if type(vl)=='table' then vl=tableConcat(vl) end
            arr[1+#arr] = vl:gsub('^%s?%s?','  '):gsub('^\n','')
         end
         arr[1+#arr] = '\n'
         arr[1+#arr] = "#endmacro"
      end
      return tableConcat(arr)
   end
end

-- This function dumps all macros to file descriptor outputfile

local function dumpMacros(macros, outputfile)
   outputfile = outputfile or io.output()
   assert(type(macros) == 'table')
   assert(io.type(outputfile) == 'file')
   for k,_ in pairs(macros) do
      local s = macroToString(macros,k)
      if s then outputfile:write(string.format("%s\n", s)) end
   end
end



-- A coroutine that filters out spaces, directives, and magic tokens

local function filterSpaces(options, tokens, ...)
   local ti = wrap(options, tokens, ...)
   local tok,n = ti()
   while tok do
      -- skip directives
      while isNewline(tok) do
         tok,n = ti()
         while isBlank(tok) do tok,n = ti() end
         if tok == '#' then
            while not isNewline(tok) do
               tok, n = ti()
            end
         end
      end
      -- output nonspaces
      if not isBlank(tok) then
         coroutine.yield(tok, n)
      end
      tok,n = ti()
   end
end

-- This function takes a line iterator and an optional location prefix.
-- It returns a token iterator for the preprocessed tokens
-- and a table of macro definitions.

local function cppTokenIterator(options, lines, prefix)
   options = copyOptions(options)
   prefix = prefix or ""
   assert(type(options)=='table')
   assert(type(lines)=='function')
   assert(type(prefix)=='string')
   local macros = initialMacros(options)
   local ti = wrap(options,
                   filterSpaces,
                   expandMacros, macros,
                   tokenize,
                   processDirectives, macros,
                   eliminateComments,
                   joinLines,
                   yieldLines, lines, prefix)
   return ti, macros
end


-- A coroutine that reconstructs lines from the preprocessed tokens

local function preprocessedLines(options, tokens, ...)
   local ti = wrap(options, tokens, ...)
   local tok,n = ti()
   while tok do
      local curn = n
      local curl = {}
      if isNewline(tok) then
         curn = n
         curl[1+#curl] = tok:sub(2)
         tok, n = ti()
      end
      while tok and not isNewline(tok) do
         if not isMagic(tok) then curl[1+#curl] = tok end
         tok, n = ti()
      end
      coroutine.yield(table.concat(curl), curn)
   end
end


-- This function preprocesses file <filename>.
-- The optional argument <outputfile> specifies where to write the
-- preprocessed file and may be a string or a file descriptor.
-- The optional argument <options> contains an array of option strings.
-- Note that option "-Zpass" is added, unless the option "-Znopass" is present.

--< @r-lyeh: iterator+string methods
local function cppIterator(lines, outputfile, options, prefix)
   -- handle optional arguments
   prefix = prefix or ''
   options = copyOptions(options)
   outputfile = outputfile or "-"
   assert(type(lines)=='function')
   assert(type(options)=='table')
   local closeoutputfile = false
   if io.type(outputfile) ~= 'file' then
      assert(type(outputfile) == 'string')
      if outputfile == '-' then
         outputfile = io.output()
      else
         closeoutputfile = true
         outputfile = io.open(outputfile,"w")
      end
   end
   assert(io.type(outputfile) == 'file')
   -- makes option -Zpass on by default
   if not hasOption(options,"-Znopass") then
      options.hash["-Zpass"] = true
   end
   -- prepare iterator
   local dM = hasOption(options, "-dM")
   local macros = initialMacros(options)
   local li = wrap(options,
                   preprocessedLines,
                   expandMacros, macros,
                   tokenize,
                   processDirectives, macros,
                   eliminateComments,
                   joinLines,
                   yieldLines, lines, prefix)
   -- iterate, inserting line markers
   local lm = hasOption(options,"-Zpass") and "line" or ""
   local cf, cn
   for s,n in li do
      if not dM and s:find("[^%s]") then
         local xf, xn
         if type(n) == 'number' then
            xn = n
         elseif type(n) == 'string' then
            xf, xn = n:match("^([^:]*).-(%d*)$")
            xn = tonumber(xn)
         end
         if cf ~= xf or cn ~= xn then
            cf, cn = xf, xn
            if lm ~= "" then --< @r-lyeh: do not output empty #line directives: ie, `#line 19 "file.c"` -> `# 19 ""`
            outputfile:write(string.format("#%s %d %q\n", lm, cn, cf))
            end
         end
         outputfile:write(s)
         outputfile:write("\n")
         cn = cn + 1
      end
   end
   if dM then
      dumpMacros(macros, outputfile)
   end
   if closeoutputfile then outputfile:close() end
end

local function cppString(str, outputfile, options)
   assert(type(str)=='string')

   return cppIterator(str:gmatch("[^\r\n]+"), outputfile, options)
end

local function cpp(filename, outputfile, options)
   assert(type(filename)=='string')

   return cppIterator(io.lines(filename), outputfile, options, filename)
end
--<

---------------------------------------------------
---------------------------------------------------
---------------------------------------------------
-- EXPORTS

cparser = {}

cparser.cpp = cpp
cparser.cppTokenIterator = cppTokenIterator
cparser.macroToString = macroToString

--< @r-lyeh: add string methods
cparser.cppString = cppString
cparser.parseString = parseString
--<

return cparser