handling of string escape sequences in the lexer: not being fooled by double-backslas...

[metalua.git] / src / compiler / lexer.lua

diff --git a/src/compiler/lexer.lua b/src/compiler/lexer.lua
index 8c4752f7b597e55273286746059157e9d2a2dc62..698e5ddccf49b6382554e190f80a899f493e1348 100644 (file)
--- a/src/compiler/lexer.lua
+++ b/src/compiler/lexer.lua
@@ -67,8 +67,13 @@ local function unescape_string (s)
    -- Turn the digits of an escape sequence into the corresponding
    -- character, e.g. [unesc_digits("123") == string.char(123)].
    local function unesc_digits (x)
+      if x:sub(1,1)=="\\" then return x end -- Hack to parse correctly "\\123"
       local k, j, i = x:reverse():byte(1, 3)
       local z = _G.string.byte "0"
+      local code = (k or z) + 10*(j or z) + 100*(i or z) - 111*z
+      if code > 255 then 
+        error ("Illegal escape sequence '\\"..x.."' in string: ASCII codes must be in [0..255]") 
+      end
       return _G.string.char ((k or z) + 10*(j or z) + 100*(i or z) - 111*z)
    end
 
@@ -84,7 +89,7 @@ local function unescape_string (s)
 
    return s
       :gsub ("\\(%D)",unesc_letter)
-      :gsub ("\\([0-9][0-9]?[0-9]?)", unesc_digits)
+      :gsub ("\\(\\?[0-9][0-9]?[0-9]?)", unesc_digits)
 end
 
 lexer.extractors = {
@@ -96,7 +101,25 @@ lexer.token_metatable = {
 --         __tostring = function(a) 
 --            return string.format ("`%s{'%s'}",a.tag, a[1]) 
 --         end 
-      } 
+} 
+      
+lexer.lineinfo_metatable = { }
+--[[ 
+-- The presence of this function prevents serialization by Pluto, 
+-- I can't figure out why :(
+function lexer.lineinfo_metatable:__tostring()
+   local txt = string.format("%s:%i(%i,%i)", self[4], self[3], self[1], self[2])
+   if self.comments then 
+      acc = { }
+      for comment in ivalues(self.comments) do
+        local content, loc1, loc2, kind = unpack(comment)
+        table.insert (acc, string.format ("%s@%i..%i:%q", kind, loc1, loc2, content))
+      end
+      txt = txt.."["..table.concat(acc,"; ").."]"
+   end
+   return txt
+end
+--]]
 
 ----------------------------------------------------------------------
 -- Really extract next token fron the raw string 
@@ -135,6 +158,9 @@ function lexer:extract ()
       -- lineinfo entries: [1]=line, [2]=column, [3]=char, [4]=filename
       local fli = { first_line, loc-first_column_offset, loc, self.src_name }
       local lli = { self.line, self.i-self.column_offset-1, self.i-1, self.src_name }
+      --Pluto barfes when the metatable is set:(
+      setmetatable(fli, lexer.lineinfo_metatable)
+      setmetatable(lli, lexer.lineinfo_metatable)
       local a = { tag = tag, lineinfo = { first=fli, last=lli }, content } 
       if lli[2]==-1 then lli[1], lli[2] = lli[1]-1, previous_line_length-1 end
       if #self.attached_comments > 0 then