Added filename reference in lineinfo, fixed positions reporting

diff --git a/src/compiler/gg.lua b/src/compiler/gg.lua
index c3fc8bce81d9ad144e2ca90b36b67a1b907481cb..08dd465af130d039d7fcd6c11411b132bdb6f1b8 100644 (file)
--- a/src/compiler/gg.lua
+++ b/src/compiler/gg.lua
@@ -42,7 +42,7 @@ module("gg", package.seeall)
 local parser_metatable = { }\r
 function parser_metatable.__call (parser, lx, ...)\r
    --printf ("Call parser %q of type %q", parser.name or "?", parser.kind)\r
-   if true or mlc.metabugs then \r
+   if mlc.metabugs then \r
       return parser:parse (lx, ...) \r
       --local x = parser:parse (lx, ...) \r
       --printf ("Result of parser %q: %s", \r
@@ -50,7 +50,7 @@ function parser_metatable.__call (parser, lx, ...)
       --        _G.table.tostring(x, "nohash", 80))\r
       --return x\r
    else\r
-      local li = lx:lineinfo().first or { "?", "?", "?" }\r
+      local li = lx:lineinfo_right() or { "?", "?", "?", "?" }\r
       local status, ast = pcall (parser.parse, parser, lx, ...)      \r
       if status then return ast else\r
          error (string.format ("%s\n - (l.%s, c.%s, k.%s) in parser %s", \r
@@ -120,10 +120,9 @@ local function transform (ast, parser, fli, lli)
    if parser.transformers then\r
       for _, t in ipairs (parser.transformers) do ast = t(ast) or ast end\r
    end\r
-   -- FIXME: add source info in lineinfo\r
-   if type(ast) == 'table' and not ast.lineinfo then\r
+   if type(ast) == 'table'then\r
       ast.lineinfo = { first=fli, last=lli }\r
-      if ast[1] and ast[1].lineinfo and ast[1].lineinfo.comments then\r
+      if type(ast[1]) == 'table' and ast[1].lineinfo and ast[1].lineinfo.comments then\r
          ast.lineinfo.comments = ast[1].lineinfo.comments\r
          table.print (ast.lineinfo.comments)\r
       end\r
@@ -135,7 +134,7 @@ end
 -- Generate a tracable parsing error (not implemented yet)\r
 -------------------------------------------------------------------------------\r
 function parse_error(lx, fmt, ...)\r
-   local li = lx:peek().lineinfo.first or {-1,-1,-1}\r
+   local li = lx:lineinfo_left() or {-1,-1,-1, "<unknown file>"}\r
    local msg  = string.format("line %i, char %i: "..fmt, li[1], li[2], ...)   \r
    local src = lx.src\r
    if li[3]>0 and src then\r
@@ -184,9 +183,9 @@ function sequence (p)
    -------------------------------------------------------------------\r
    function p:parse (lx)\r
       -- Raw parsing:\r
-      local fli = lx:lineinfo()\r
+      local fli = lx:lineinfo_right()\r
       local seq = raw_parse_sequence (lx, self)\r
-      local lli = lx:lineinfo()\r
+      local lli = lx:lineinfo_left()\r
 \r
       -- Builder application:\r
       local builder, tb = self.builder, type (self.builder)\r
@@ -290,9 +289,9 @@ function multisequence (p)
    -- Parsing method\r
    -------------------------------------------------------------------\r
    function p:parse (lx)\r
-      local fli = lx:lineinfo()\r
+      local fli = lx:lineinfo_right()\r
       local x = raw_parse_multisequence (lx, self.sequences, self.default)\r
-      local lli = lx:lineinfo()\r
+      local lli = lx:lineinfo_left()\r
       return transform (x, self, fli, lli)\r
    end\r
 \r
@@ -392,17 +391,17 @@ function expr (p)
       -- expr, and one for the one with the prefix op.\r
       ------------------------------------------------------\r
       local function handle_prefix ()\r
-         local fli = lx:lineinfo()\r
+         local fli = lx:lineinfo_right()\r
          local p2_func, p2 = get_parser_info (self.prefix)\r
          local op = p2_func and p2_func (lx)\r
          if op then -- Keyword-based sequence found\r
-            local ili = lx:lineinfo() -- Intermediate LineInfo\r
+            local ili = lx:lineinfo_right() -- Intermediate LineInfo\r
             local e = p2.builder (op, self:parse (lx, p2.prec))\r
-            local lli = lx:lineinfo()\r
+            local lli = lx:lineinfo_left()\r
             return transform (transform (e, p2, ili, lli), self, fli, lli)\r
          else -- No prefix found, get a primary expression         \r
             local e = self.primary(lx)\r
-            local lli = lx:lineinfo()            \r
+            local lli = lx:lineinfo_left()\r
             return transform (e, self, fli, lli)\r
          end\r
       end --</expr.parse.handle_prefix>\r
@@ -423,7 +422,7 @@ function expr (p)
          -- return.\r
          -----------------------------------------\r
          if (not p2.prec or p2.prec>prec) and p2.assoc=="flat" then\r
-            local fli = lx:lineinfo()\r
+            local fli = lx:lineinfo_right()\r
             local pflat, list = p2, { e }\r
             repeat\r
                local op = p2_func(lx)\r
@@ -433,7 +432,7 @@ function expr (p)
                _, p2 = get_parser_info (self.infix)\r
             until p2 ~= pflat\r
             local e2 = pflat.builder (list)\r
-            local lli = lx:lineinfo()\r
+            local lli = lx:lineinfo_left()\r
             return transform (transform (e2, pflat, fli, lli), self, fli, lli)\r
  \r
          -----------------------------------------\r
@@ -443,12 +442,12 @@ function expr (p)
          -----------------------------------------\r
          elseif p2.prec and p2.prec>prec or \r
                 p2.prec==prec and p2.assoc=="right" then\r
-            local fli = lx:lineinfo()\r
+            local fli = lx:lineinfo_right()\r
             local op = p2_func(lx)\r
             if not op then return false end\r
             local e2 = self:parse (lx, p2.prec)\r
             local e3 = p2.builder (e, op, e2)\r
-            local lli = lx:lineinfo()\r
+            local lli = lx:lineinfo_left()\r
             return transform (transform (e3, p2, fli, lli), self, fli, lli)\r
 \r
          -----------------------------------------\r
@@ -474,10 +473,10 @@ function expr (p)
          local p2_func, p2 = get_parser_info (self.suffix)\r
          if not p2 then return false end\r
          if not p2.prec or p2.prec>=prec then\r
-            local fli = lx:lineinfo()\r
+            local fli = lx:lineinfo_right()\r
             local op = p2_func(lx)\r
             if not op then return false end\r
-            local lli = lx:lineinfo()\r
+            local lli = lx:lineinfo_left()\r
             e = p2.builder (e, op)\r
             e = transform (transform (e, p2, fli, lli), self, fli, lli)\r
             return e\r
@@ -559,7 +558,7 @@ function list (p)
          return keywords and lx:is_keyword(lx:peek(), unpack(keywords)) end\r
 \r
       local x = { }\r
-      local fli = lx:lineinfo()\r
+      local fli = lx:lineinfo_right()\r
 \r
       -- if there's a terminator to start with, don't bother trying\r
       if not peek_is_in (self.terminators) then \r
@@ -574,7 +573,7 @@ function list (p)
             lx:peek().tag=="Eof"\r
       end\r
 \r
-      local lli = lx:lineinfo()\r
+      local lli = lx:lineinfo_left()\r
       \r
       -- Apply the builder. It can be a string, or a callable value, \r
       -- or simply nothing.\r
@@ -645,9 +644,9 @@ function onkeyword (p)
    -------------------------------------------------------------------\r
    function p:parse(lx)\r
       if lx:is_keyword (lx:peek(), unpack(self.keywords)) then\r
-         local fli = lx:lineinfo()\r
+         local fli = lx:lineinfo_right()\r
          if not self.peek then lx:next() end\r
-         local lli = lx:lineinfo()\r
+         local lli = lx:lineinfo_left()\r
          return transform (self.primary(lx), p, fli, lli)\r
       else return false end\r
    end\r

diff --git a/src/compiler/lexer.lua b/src/compiler/lexer.lua
index c825caab6187370f6b405f8066ecfaa8b41f8c64..6815011ec79855d950c5ed2cd447d4a33ae67b5f 100644 (file)
--- a/src/compiler/lexer.lua
+++ b/src/compiler/lexer.lua
@@ -111,30 +111,32 @@ function lexer:extract ()
 
    -- Put line info, comments and metatable arount the tag and content
    -- provided by extractors, thus returning a complete lexer token.
+   -- first_line: line # at the beginning of token
+   -- first_column_offset: char # of the last '\n' before beginning of token
+   -- i: scans from beginning of prefix spaces/comments to end of token.
    local function build_token (tag, content)
       assert (tag and content)
-      local i, first_line, first_column_offset =
-         previous_i, self.line, self.column_offset
+      local i, first_line, first_column_offset, previous_line_length =
+         previous_i, self.line, self.column_offset, nil
+
       -- update self.line and first_line. i := indexes of '\n' chars
       while true do
          i = self.src :find ("\n", i+1, true)
-         if not i then break end
-         if loc and i <= loc then 
+         if not i or i>self.i then break end -- no more '\n' until end of token
+         previous_line_length = i - self.column_offset
+         if loc and i <= loc then -- '\n' before beginning of token
             first_column_offset = i
             first_line = first_line+1 
          end
-         if i <= self.i then
-            self.line   = self.line+1 
-            self.column_offset = i 
-         else break end
+         self.line   = self.line+1 
+         self.column_offset = i 
       end
-      local a = { --char = loc, line = self.line,
-         tag      = tag, 
-         lineinfo = {
-            name  = self.src_name,
-            first = { first_line, loc - first_column_offset, loc }, 
-            last  = { self.line,  self.i - self.column_offset, self.i } },
-         content } 
+
+      -- lineinfo entries: [1]=line, [2]=column, [3]=char, [4]=filename
+      local fli = { first_line, loc-first_column_offset, loc, self.src_name }
+      local lli = { self.line, self.i-self.column_offset-1, self.i-1, self.src_name }
+      local a = { tag = tag, lineinfo = { first=fli, last=lli }, content } 
+      if lli[2]==-1 then lli[1], lli[2] = lli[1]-1, previous_line_length-1 end
       if #self.attached_comments > 0 then 
          a.lineinfo.comments = self.attached_comments 
          self.attached_comments = nil
@@ -343,6 +345,7 @@ function lexer:next (n)
       end
       self.lastline = a.lineinfo.last[1]
    end
+   self.lineinfo_last = a.lineinfo.last
    return a or eof_token
 end
 
@@ -383,9 +386,30 @@ function lexer:takeover(old)
    return self
 end
 
-function lexer:lineinfo()
-       if self.peeked[1] then return self.peeked[1].lineinfo.first
-    else return { self.line, self.i-self.column_offset, self.i } end
+-- function lexer:lineinfo()
+--     if self.peeked[1] then return self.peeked[1].lineinfo.first
+--     else return { self.line, self.i-self.column_offset, self.i } end
+-- end
+
+
+----------------------------------------------------------------------
+-- Return the current position in the sources. This position is between
+-- two tokens, and can be within a space / comment area, and therefore
+-- have a non-null width. :lineinfo_left() returns the beginning of the
+-- separation area, :lineinfo_right() returns the end of that area.
+--
+--    ____ last consummed token    ____ first unconsummed token
+--   /                            /
+-- XXXXX  <spaces and comments> YYYYY
+--      \____                    \____
+--           :lineinfo_left()         :lineinfo_right()
+----------------------------------------------------------------------
+function lexer:lineinfo_right()
+   return self:peek(1).lineinfo.first
+end
+
+function lexer:lineinfo_left()
+   return self.lineinfo_last
 end
 
 ----------------------------------------------------------------------
@@ -404,7 +428,8 @@ function lexer:newstream (src_or_stream, name)
          i             = 1;      -- Character offset in src
          line          = 1;      -- Current line number
          column_offset = 0;      -- distance from beginning of file to last '\n'
-         attached_comments = { } -- comments accumulator
+         attached_comments = { },-- comments accumulator
+         lineinfo_last = { 1, 1, 1, name }
       }
       setmetatable (stream, self)