Initial check in lib

2026-01-03 18:33:48 +01:00
parent b20cd8e688
commit 5666f85e99
89 changed files with 36370 additions and 1 deletions
@@ -0,0 +1,110 @@
+unit dpglib.TokenLexer;
+
+lexer TTokenLexer;
+options
+{
+   k              = 2;
+   testLiterals   = false;
+}
+
+// ----------------------------------------------------------------------------
+// Simple tokens
+// ----------------------------------------------------------------------------
+LPAREN   :  '(';
+RPAREN   :  ')';
+ASSIGN   :  '=';
+STRING   :  '"' (~'"')* '"';
+
+// ----------------------------------------------------------------------------
+// DIGIT
+// ----------------------------------------------------------------------------
+protected
+DIGIT
+   :  '0'..'9'
+   ;
+   
+// ----------------------------------------------------------------------------
+// XDIGIT
+// ----------------------------------------------------------------------------
+protected
+XDIGIT
+   :  '0'..'9'
+   |  'a'..'f'
+   |  'A'..'F'
+   ;
+
+// ----------------------------------------------------------------------------
+// ID
+// ----------------------------------------------------------------------------
+ID
+   :  ('a'..'z' | 'A'..'Z')
+      ('a'..'z' | 'A'..'Z' | '_' | '0'..'9')*
+   ;
+   
+// ----------------------------------------------------------------------------
+// INT
+// ----------------------------------------------------------------------------
+INT
+   : (DIGIT)+
+   ;
+   
+// ----------------------------------------------------------------------------
+// WS
+// ----------------------------------------------------------------------------
+WS
+   :
+   (
+      ' '
+   |  '\t'
+   |  '\r' '\n'   { newLine; }
+   |  '\r'        { newLine; }
+   |  '\n'        { newLine; }
+   )
+   {
+      _ttype := TT_SKIP;
+   }
+   ;
+      
+// ----------------------------------------------------------------------------
+// SLCOMMENT
+// ----------------------------------------------------------------------------
+SLCOMMENT
+         :  "//"
+            ( ~( '\r' | '\n'))*
+            (
+               options
+               {
+                  generateAmbigWarnings = false;
+               }
+            :  '\r' '\n'      { newLine;  }
+            |  '\r'           { newLine;  }
+            |  '\n'           { newLine;  }
+            )
+            {
+               _ttype := TT_SKIP;
+            }
+         ;
+      
+// ----------------------------------------------------------------------------
+// MLCOMMENT
+// ----------------------------------------------------------------------------
+MLCOMMENT
+         :  "(*"
+            
+            (
+               options
+               {
+                  greedy               = false;
+                  generateAmbigWarnings= false;
+               }
+            :  '\r' '\n'   { newLine;  }
+            |  '\r'        { newLine;  }
+            |  '\n'        { newLine;  }
+            |  ~('\r' | '\n')
+            )*
+            
+            "*)"
+            {
+               _ttype := TT_SKIP;
+            }
+         ;