+----------------------------------------------------------------------
+-- Resynchronize: cancel any token in self.peeked, by emptying the
+-- list and resetting the indexes
+----------------------------------------------------------------------
+function lexer:sync()
+ local p1 = self.peeked[1]
+ if p1 then
+ li = p1.lineinfo.first
+ self.line, self.i = li[1], li[3]
+ self.column_offset = self.i - li[2]
+ self.peeked = { }
+ self.attached_comments = p1.lineinfo.first.comments or { }
+ end
+end
+
+----------------------------------------------------------------------
+-- Take the source and offset of an old lexer.
+----------------------------------------------------------------------
+function lexer:takeover(old)
+ self:sync()
+ self.line, self.column_offset, self.i, self.src, self.attached_comments =
+ old.line, old.column_offset, old.i, old.src, old.attached_comments
+ return self
+end
+
+-- function lexer:lineinfo()
+-- if self.peeked[1] then return self.peeked[1].lineinfo.first
+-- else return { self.line, self.i-self.column_offset, self.i } end
+-- end
+
+
+----------------------------------------------------------------------
+-- Return the current position in the sources. This position is between
+-- two tokens, and can be within a space / comment area, and therefore
+-- have a non-null width. :lineinfo_left() returns the beginning of the
+-- separation area, :lineinfo_right() returns the end of that area.
+--
+-- ____ last consummed token ____ first unconsummed token
+-- / /
+-- XXXXX <spaces and comments> YYYYY
+-- \____ \____
+-- :lineinfo_left() :lineinfo_right()
+----------------------------------------------------------------------
+function lexer:lineinfo_right()
+ return self:peek(1).lineinfo.first
+end
+
+function lexer:lineinfo_left()
+ return self.lineinfo_last
+end
+