Add improvements to the python3 syntax definitions (#1833)

* Python3: Add built-in object 'cls'

* Python3: Add the bitwise negation operator ~

* Python3: Add support for hexadeximal and binary numerical literals

* Python3: Rewrite '(__foo__|__bar__)' as '__(foo|bar)__', add known '__i.*__' methods

* Python3: Add __iter__ as a magic method, sort the list of magic methods

* Python3: Numerical literals: Add support for '_', disallow leading 0 for decimals

* fixup! Python3: Numerical literals: Add support for '_', disallow leading 0 for decimals

hex oct and bin have different sets of allowed digits

* Python3: Add support for floating point numbers with optional scientific notation

* Python3: stop single-line strings at EOL

* Python3: Add support for TODO and FIXME in comments

* Python3: Add support for the ^ bitwise xor operator

diff --git a/runtime/syntax/python3.yaml b/runtime/syntax/python3.yaml
index a3af2abbcb54528abd7092e52ab33061235b97fb..0afcd051a305d7b93884bd0253d727e6b37d9c17 100644 (file)
--- a/runtime/syntax/python3.yaml
+++ b/runtime/syntax/python3.yaml
@@ -6,13 +6,13 @@ detect:
 
 rules:
     # built-in objects
-    - constant: "\\b(Ellipsis|None|self|True|False)\\b"
+    - constant: "\\b(Ellipsis|None|self|cls|True|False)\\b"
       # built-in attributes
     - constant: "\\b(__bases__|__builtin__|__class__|__debug__|__dict__|__doc__|__file__|__members__|__methods__|__name__|__self__)\\b"
       # built-in functions
     - identifier: "\\b(abs|all|any|ascii|bin|bool|breakpoint|bytearray|bytes|callable|chr|classmethod|compile|complex|delattr|dir|divmod|eval|exec|format|getattr|globals|hasattr|hash|help|hex|id|input|isinstance|issubclass|iter|len|locals|max|min|next|nonlocal|oct|open|ord|pow|print|repr|round|setattr|sorted|sum|vars|__import__)\\b"
       # special method names
-    - identifier: "\\b(__abs__|__add__|__and__|__call__|__cmp__|__coerce__|__complex__|__concat__|__contains__|__del__|__delattr__|__delitem__|__delslice__|__div__|__divmod__|__float__|__getattr__|__getitem__|__getslice__|__hash__|__hex__|__init__|__int__|__inv__|__invert__|__len__|__dict__|__long__|__lshift__|__mod__|__mul__|__neg__|__next__|__nonzero__|__oct__|__or__|__pos__|__pow__|__radd__|__rand__|__rcmp__|__rdiv__|__rdivmod__|__repeat__|__repr__|__rlshift__|__rmod__|__rmul__|__ror__|__rpow__|__rrshift__|__rshift__|__rsub__|__rxor__|__setattr__|__setitem__|__setslice__|__str__|__sub__|__xor__)\\b"
+    - identifier: "\\b__(abs|add|and|call|cmp|coerce|complex|concat|contains|delattr|delitem|delslice|del|dict|divmod|div|float|getattr|getitem|getslice|hash|hex|iadd|iand|iconcat|ifloordiv|ilshift|imatmul|imod|imul|init|int|invert|inv|ior|ipow|irshift|isub|iter|itruediv|ixor|len|long|lshift|mod|mul|neg|next|nonzero|oct|or|pos|pow|radd|rand|rcmp|rdivmod|rdiv|repeat|repr|rlshift|rmod|rmul|ror|rpow|rrshift|rshift|rsub|rxor|setattr|setitem|setslice|str|sub|xor)__\\b"
       # types
     - type: "\\b(bool|bytearray|bytes|classmethod|complex|dict|enumerate|filter|float|frozenset|int|list|map|memoryview|object|property|range|reversed|set|slice|staticmethod|str|super|tuple|type|zip)\\b"
       # definitions
@@ -22,11 +22,14 @@ rules:
       # decorators
     - brightgreen: "@.*[(]"
       # operators
-    - symbol.operator: "([.:;,+*|=!\\%@]|<|>|/|-|&)"
+    - symbol.operator: "([~^.:;,+*|=!\\%@]|<|>|/|-|&)"
       # parentheses
     - symbol.brackets: "([(){}]|\\[|\\])"
       # numbers
-    - constant.number: "\\b[0-9]+\\b"
+    - constant.number: "\\b[1-9](_?[0-9])*(\\.([0-9](_?[0-9])*)?)?(e[0-9](_?[0-9])*)?\\b" # decimal
+    - constant.number: "\\b0b(_?[01])+\\b"     # bin
+    - constant.number: "\\b0o(_?[0-7])+\\b"    # oct
+    - constant.number: "\\b0x(_?[0-9a-f])+\\b" # hex
 
     - constant.string:
         start: "\"\"\""
@@ -40,14 +43,14 @@ rules:
 
     - constant.string:
         start: "\""
-        end: "\""
+        end: "(\"|$)"
         skip: "\\\\."
         rules:
             - constant.specialChar: "\\\\."
 
     - constant.string:
         start: "'"
-        end: "'"
+        end: "('|$)"
         skip: "\\\\."
         rules:
             - constant.specialChar: "\\\\."
@@ -55,5 +58,5 @@ rules:
     - comment:
         start: "#"
         end: "$"
-        rules: []
-
+        rules:
+            - todo: "(TODO|FIXME):?"