bds.mr.dpg/src.lib/grammar/test/Lexer.g

unit z.dpglib.DpgLexer;

lexer TDpgLexer;
options
{
   testLiterals = false;
   k            = 2;
}

tokens
{
   "unit";
   "uses";
   "const";
   "type";
   
   "lexer";
   "parser";
   
   "options";
   "tokens";
   "memberdecl";
   "memberdef";
   
   "private";
   "protected";
   "public";
   
   "returns";
   "local";
   
   "except";
   "finally";
   
   SEMPRED;
   
   USES;
   OPTIONS;
   TOKENS;
}

// ----------------------------------------------------------------------------
// Simple tokens
// ----------------------------------------------------------------------------
LPAREN:     '(';
RPAREN:     ')';

RCURLY:     '}';

COLON:      ':';
SEMI:       ';';
COMMA:      ',';

ASSIGN:     '=';
IMPLIES:    "=>";

QUEST:      '?';
PLUS:       '+';
STAR:       '*';
AT:         '@';

NOT:        '~';
OR:         '|';
BANG:       '!';

WILDCARD:   '.';
RANGE:      "..";

// ----------------------------------------------------------------------------
// Character literal
// ----------------------------------------------------------------------------
CHARLIT
   :  '\''! (ESC | ~'\'') '\''! ;

// ----------------------------------------------------------------------------
// String literal
// ----------------------------------------------------------------------------
STRINGLIT
   :  '"' (ESC | ~'"')* '"' ;

// ----------------------------------------------------------------------------
// Integer
// ----------------------------------------------------------------------------
INTEGER
local
{  
   i: integer;
   v: integer;
}
   :
      (
         DNUMBER
         {
            v := 0;
            for i:=1 to Length( TokenText) do
            begin
               v := v * 10 + ord( TokenText[i]) - ord('0');
            end;
               
            TokenText := IntToStr( v);
         }
      |
         XNUMBER
         {
            v := 0;
            for i:=1 to Length( TokenText) do
            begin
               case TokenText[i] of
                  '0'..'9':   v := v * 16 + ord(TokenText[i]) - ord('0');
                  'a'..'z':   v := v * 16 + ord(TokenText[i]) - ord('a');
                  'A'..'Z':   v := v * 16 + ord(TokenText[i]) - ord('A');
               end;
            end;
         
            TokenText := IntToStr( v);
         }
      )
   ;
      

// ----------------------------------------------------------------------------
// Argument action
// ----------------------------------------------------------------------------
ARGACTION
   :
      '['!
      (
         options
         {  
            generateAmbigWarnings = false;
         }
         : '\r' '\n'   {  newLine; }
         | '\r'        {  newLine; }
         | '\n'        {  newLine; }
         | ~']'
      )*
      ']'!
   ;
   
// ----------------------------------------------------------------------------
// Action
// ----------------------------------------------------------------------------
ACTION
   :
      '{'
      (
         options
         {
            generateAmbigWarnings = false;
         }     
         : '\r' '\n' {  newLine; }
         | '\r'      {  newLine; }
         | '\n'      {  newLine; }
         | ~'}'
      )*
      '}'
      ( '?'! { _ttype := TT_SEMPRED; } )?
   ;
   
// ----------------------------------------------------------------------------
// Token ref
// ----------------------------------------------------------------------------
TOKENREF
options
{
   testLiterals = true;
}
   :  'A'..'Z' ('a'..'z' | 'A'..'Z' | '_' | '0'..'9')* ;
   
// ----------------------------------------------------------------------------
// Rule ref
// ----------------------------------------------------------------------------
RULEREF
local
{
   t: integer;
}
   :
      t = INT_RULEREF { _ttype := t; }
      (
         {t = LT_uses}?     WS_LOOP ('{'  { _ttype := TT_USES;    } )?
      |  {t = LT_options}?  WS_LOOP ('{'  { _ttype := TT_OPTIONS; } )?
      |  {t = LT_tokens}?   WS_LOOP ('{'  { _ttype := TT_TOKENS;  } )?
      )?
   ;
   
// ----------------------------------------------------------------------------
// Internal rule ref
// ----------------------------------------------------------------------------
protected
INT_RULEREF returns [integer]
{
   _ttype := TT_RULEREF;
}
   :  'a'..'z' ('a'..'z' | 'A'..'Z' | '_' | '0'..'9')* 
   {
      result := TestLiteral( _ttype);
   }
   ;
   
// ----------------------------------------------------------------------------
// COMMENT
// ----------------------------------------------------------------------------
COMMENT
   :  SLCOMMENT   { _ttype := TT_SKIP; }
   |  MLCOMMENT1  { _ttype := TT_SKIP; }
   |  MLCOMMENT2  { _ttype := TT_SKIP; }
   ;
   
// ----------------------------------------------------------------------------
// SLCOMMENT
// ----------------------------------------------------------------------------
protected
SLCOMMENT
   :
      "//"
      ( ~( '\r' | '\n') )*
      (
         options
         {
            generateAmbigWarnings = false;
         }
         : '\r' '\n' {  newLine; }
         | '\r'      {  newLine; }
         | '\n'      {  newLine; }
      )
   ;   

// ============================================================================
// Multi line comment version 1
// Nested comments aren't allowed!
// ============================================================================
protected
MLCOMMENT1
   :
      "(*"
      (
         options 
         {
            greedy = false;
            generateAmbigWarnings = false;
         }
         :  '\r' '\n'   {  newLine; }
         |  '\r'        {  newLine; }
         |  '\n'        {  newLine; }
         |  .
      )*
      "*)"
   ;
   
// ============================================================================
// Multi line comment version 2
// Nested comments aren't allowed!
// ============================================================================
protected
MLCOMMENT2
   :
      "/*"
      (
         options 
         {
            greedy = false;
            generateAmbigWarnings = false;
         }
         :  '\r' '\n'   {  newLine; }
         |  '\r'        {  newLine; }
         |  '\n'        {  newLine; }
         |  .
      )*
      "*/"
   ;
   
// ----------------------------------------------------------------------------
// Numbers
// ----------------------------------------------------------------------------
protected   DNUMBER: '0'..'9' (DDIGIT)*;
protected   XNUMBER: '$'!     (XDIGIT)+;

// ----------------------------------------------------------------------------
// Digits
// ----------------------------------------------------------------------------
protected   DDIGIT:  '0'..'9';
protected   XDIGIT:  '0'..'9' | 'a'..'f' | 'A'..'F';

// ----------------------------------------------------------------------------
// WS
// ----------------------------------------------------------------------------
WS
   :
      (  
         options
         {
            generateAmbigWarnings = false;
         }
         : ' '
         | '\t'      { tab;         }
         | '\r' '\n' { newLine;     }
         | '\r'      { newLine;     }
         | '\n'      { newLine;     }
      )
      {
         _ttype := TT_SKIP;
      }
   ;
   
// ----------------------------------------------------------------------------
// WS_LOOP
// ----------------------------------------------------------------------------
protected
WS_LOOP
   :
      (
         options
         {
            greedy = true;
         }
         : WS
         | COMMENT
      )*
   ;

   
   
// ----------------------------------------------------------------------------
// Esc
// ----------------------------------------------------------------------------
protected
ESC
local
{
   number: AnsiString;
}
   :  
      '\\'!
      (
         'r'   {  TokenText[ Length( TokenText)] := AnsiChar(13); }
      |  'n'   {  TokenText[ Length( TokenText)] := AnsiChar(10); }
      |  't'   {  TokenText[ Length( TokenText)] := AnsiChar(9);  }
      |  '\\'
      |  '\''
      |  '"'
      |  'x' d1:XDIGIT! d2:XDIGIT!
               {  
                  number := '$' + d1.TokenText + d2.TokenText;
                  TokenText[ Length( TokenText)] := AnsiChar( StrToInt( number));
               }
      )
   ;