gcc/d/dmd/tokens.c

   1
   2 /* Compiler implementation of the D programming language
   3  * Copyright (C) 1999-2021 by The D Language Foundation, All Rights Reserved
   4  * written by Walter Bright
   5  * http://www.digitalmars.com
   6  * Distributed under the Boost Software License, Version 1.0.
   7  * http://www.boost.org/LICENSE_1_0.txt
   8  * https://github.com/D-Programming-Language/dmd/blob/master/src/lexer.c
   9  */
  10
  11 #include "root/dsystem.h"
  12
  13 #include "tokens.h"
  14 #include "root/rmem.h"
  15 #include "root/outbuffer.h"
  16 #include "id.h"
  17 #include "identifier.h"
  18 #include "utf.h"
  19
  20 /************************* Token **********************************************/
  21
  22 Token *Token::freelist = NULL;
  23
  24 const char *Token::tochars[TOKMAX];
  25
  26 Token *Token::alloc()
  27 {
  28     if (Token::freelist)
  29     {
  30         Token *t = freelist;
  31         freelist = t->next;
  32         t->next = NULL;
  33         return t;
  34     }
  35
  36     return new Token();
  37 }
  38
  39 void Token::free()
  40 {
  41     next = freelist;
  42     freelist = this;
  43 }
  44
  45 const char *Token::toChars() const
  46 {
  47     static char buffer[3 + 3 * sizeof(floatvalue) + 1];
  48
  49     const char *p = &buffer[0];
  50     switch (value)
  51     {
  52         case TOKint32v:
  53             sprintf(&buffer[0],"%d",(d_int32)int64value);
  54             break;
  55
  56         case TOKuns32v:
  57         case TOKcharv:
  58         case TOKwcharv:
  59         case TOKdcharv:
  60             sprintf(&buffer[0],"%uU",(d_uns32)uns64value);
  61             break;
  62
  63         case TOKint64v:
  64             sprintf(&buffer[0],"%lldL",(longlong)int64value);
  65             break;
  66
  67         case TOKuns64v:
  68             sprintf(&buffer[0],"%lluUL",(ulonglong)uns64value);
  69             break;
  70
  71         case TOKfloat32v:
  72             CTFloat::sprint(&buffer[0], 'g', floatvalue);
  73             strcat(&buffer[0], "f");
  74             break;
  75
  76         case TOKfloat64v:
  77             CTFloat::sprint(&buffer[0], 'g', floatvalue);
  78             break;
  79
  80         case TOKfloat80v:
  81             CTFloat::sprint(&buffer[0], 'g', floatvalue);
  82             strcat(&buffer[0], "L");
  83             break;
  84
  85         case TOKimaginary32v:
  86             CTFloat::sprint(&buffer[0], 'g', floatvalue);
  87             strcat(&buffer[0], "fi");
  88             break;
  89
  90         case TOKimaginary64v:
  91             CTFloat::sprint(&buffer[0], 'g', floatvalue);
  92             strcat(&buffer[0], "i");
  93             break;
  94
  95         case TOKimaginary80v:
  96             CTFloat::sprint(&buffer[0], 'g', floatvalue);
  97             strcat(&buffer[0], "Li");
  98             break;
  99
 100         case TOKstring:
 101         {
 102             OutBuffer buf;
 103             buf.writeByte('"');
 104             for (size_t i = 0; i < len; )
 105             {
 106                 unsigned c;
 107                 utf_decodeChar((utf8_t *)ustring, len, &i, &c);
 108                 switch (c)
 109                 {
 110                     case 0:
 111                         break;
 112
 113                     case '"':
 114                     case '\\':
 115                         buf.writeByte('\\');
 116                         /* fall through */
 117                     default:
 118                         if (c <= 0x7F)
 119                         {
 120                             if (isprint(c))
 121                                 buf.writeByte(c);
 122                             else
 123                                 buf.printf("\\x%02x", c);
 124                         }
 125                         else if (c <= 0xFFFF)
 126                             buf.printf("\\u%04x", c);
 127                         else
 128                             buf.printf("\\U%08x", c);
 129                         continue;
 130                 }
 131                 break;
 132             }
 133             buf.writeByte('"');
 134             if (postfix)
 135                 buf.writeByte(postfix);
 136             p = buf.extractChars();
 137         }
 138             break;
 139
 140         case TOKxstring:
 141         {
 142             OutBuffer buf;
 143             buf.writeByte('x');
 144             buf.writeByte('"');
 145             for (size_t i = 0; i < len; i++)
 146             {
 147                 if (i)
 148                     buf.writeByte(' ');
 149                 buf.printf("%02x", ustring[i]);
 150             }
 151             buf.writeByte('"');
 152             if (postfix)
 153                 buf.writeByte(postfix);
 154             buf.writeByte(0);
 155             p = (char *)buf.extractData();
 156             break;
 157         }
 158
 159         case TOKidentifier:
 160         case TOKenum:
 161         case TOKstruct:
 162         case TOKimport:
 163         case TOKwchar: case TOKdchar:
 164         case TOKbool: case TOKchar:
 165         case TOKint8: case TOKuns8:
 166         case TOKint16: case TOKuns16:
 167         case TOKint32: case TOKuns32:
 168         case TOKint64: case TOKuns64:
 169         case TOKint128: case TOKuns128:
 170         case TOKfloat32: case TOKfloat64: case TOKfloat80:
 171         case TOKimaginary32: case TOKimaginary64: case TOKimaginary80:
 172         case TOKcomplex32: case TOKcomplex64: case TOKcomplex80:
 173         case TOKvoid:
 174             p = ident->toChars();
 175             break;
 176
 177         default:
 178             p = toChars(value);
 179             break;
 180     }
 181     return p;
 182 }
 183
 184 const char *Token::toChars(TOK value)
 185 {
 186     static char buffer[3 + 3 * sizeof(value) + 1];
 187
 188     const char *p = tochars[value];
 189     if (!p)
 190     {
 191         sprintf(&buffer[0],"TOK%d",value);
 192         p = &buffer[0];
 193     }
 194     return p;
 195 }
 196
 197 /****************************************
 198  */
 199
 200 struct Keyword
 201 {
 202     const char *name;
 203     TOK value;
 204 };
 205
 206 static size_t nkeywords;
 207 static Keyword keywords[] =
 208 {
 209     {   "this",         TOKthis         },
 210     {   "super",        TOKsuper        },
 211     {   "assert",       TOKassert       },
 212     {   "null",         TOKnull         },
 213     {   "true",         TOKtrue         },
 214     {   "false",        TOKfalse        },
 215     {   "cast",         TOKcast         },
 216     {   "new",          TOKnew          },
 217     {   "delete",       TOKdelete       },
 218     {   "throw",        TOKthrow        },
 219     {   "module",       TOKmodule       },
 220     {   "pragma",       TOKpragma       },
 221     {   "typeof",       TOKtypeof       },
 222     {   "typeid",       TOKtypeid       },
 223
 224     {   "template",     TOKtemplate     },
 225
 226     {   "void",         TOKvoid         },
 227     {   "byte",         TOKint8         },
 228     {   "ubyte",        TOKuns8         },
 229     {   "short",        TOKint16        },
 230     {   "ushort",       TOKuns16        },
 231     {   "int",          TOKint32        },
 232     {   "uint",         TOKuns32        },
 233     {   "long",         TOKint64        },
 234     {   "ulong",        TOKuns64        },
 235     {   "cent",         TOKint128,      },
 236     {   "ucent",        TOKuns128,      },
 237     {   "float",        TOKfloat32      },
 238     {   "double",       TOKfloat64      },
 239     {   "real",         TOKfloat80      },
 240
 241     {   "bool",         TOKbool         },
 242     {   "char",         TOKchar         },
 243     {   "wchar",        TOKwchar        },
 244     {   "dchar",        TOKdchar        },
 245
 246     {   "ifloat",       TOKimaginary32  },
 247     {   "idouble",      TOKimaginary64  },
 248     {   "ireal",        TOKimaginary80  },
 249
 250     {   "cfloat",       TOKcomplex32    },
 251     {   "cdouble",      TOKcomplex64    },
 252     {   "creal",        TOKcomplex80    },
 253
 254     {   "delegate",     TOKdelegate     },
 255     {   "function",     TOKfunction     },
 256
 257     {   "is",           TOKis           },
 258     {   "if",           TOKif           },
 259     {   "else",         TOKelse         },
 260     {   "while",        TOKwhile        },
 261     {   "for",          TOKfor          },
 262     {   "do",           TOKdo           },
 263     {   "switch",       TOKswitch       },
 264     {   "case",         TOKcase         },
 265     {   "default",      TOKdefault      },
 266     {   "break",        TOKbreak        },
 267     {   "continue",     TOKcontinue     },
 268     {   "synchronized", TOKsynchronized },
 269     {   "return",       TOKreturn       },
 270     {   "goto",         TOKgoto         },
 271     {   "try",          TOKtry          },
 272     {   "catch",        TOKcatch        },
 273     {   "finally",      TOKfinally      },
 274     {   "with",         TOKwith         },
 275     {   "asm",          TOKasm          },
 276     {   "foreach",      TOKforeach      },
 277     {   "foreach_reverse",      TOKforeach_reverse      },
 278     {   "scope",        TOKscope        },
 279
 280     {   "struct",       TOKstruct       },
 281     {   "class",        TOKclass        },
 282     {   "interface",    TOKinterface    },
 283     {   "union",        TOKunion        },
 284     {   "enum",         TOKenum         },
 285     {   "import",       TOKimport       },
 286     {   "mixin",        TOKmixin        },
 287     {   "static",       TOKstatic       },
 288     {   "final",        TOKfinal        },
 289     {   "const",        TOKconst        },
 290     {   "alias",        TOKalias        },
 291     {   "override",     TOKoverride     },
 292     {   "abstract",     TOKabstract     },
 293     {   "debug",        TOKdebug        },
 294     {   "deprecated",   TOKdeprecated   },
 295     {   "in",           TOKin           },
 296     {   "out",          TOKout          },
 297     {   "inout",        TOKinout        },
 298     {   "lazy",         TOKlazy         },
 299     {   "auto",         TOKauto         },
 300
 301     {   "align",        TOKalign        },
 302     {   "extern",       TOKextern       },
 303     {   "private",      TOKprivate      },
 304     {   "package",      TOKpackage      },
 305     {   "protected",    TOKprotected    },
 306     {   "public",       TOKpublic       },
 307     {   "export",       TOKexport       },
 308
 309     {   "invariant",    TOKinvariant    },
 310     {   "unittest",     TOKunittest     },
 311     {   "version",      TOKversion      },
 312
 313     {   "__argTypes",   TOKargTypes     },
 314     {   "__parameters", TOKparameters   },
 315     {   "ref",          TOKref          },
 316     {   "macro",        TOKmacro        },
 317
 318     {   "pure",         TOKpure         },
 319     {   "nothrow",      TOKnothrow      },
 320     {   "__gshared",    TOKgshared      },
 321     {   "__traits",     TOKtraits       },
 322     {   "__vector",     TOKvector       },
 323     {   "__overloadset", TOKoverloadset },
 324     {   "__FILE__",     TOKfile         },
 325     {   "__FILE_FULL_PATH__", TOKfilefullpath  },
 326     {   "__LINE__",     TOKline         },
 327     {   "__MODULE__",   TOKmodulestring },
 328     {   "__FUNCTION__", TOKfuncstring   },
 329     {   "__PRETTY_FUNCTION__", TOKprettyfunc   },
 330     {   "shared",       TOKshared       },
 331     {   "immutable",    TOKimmutable    },
 332     {   NULL,           TOKreserved     }
 333 };
 334
 335 int Token::isKeyword()
 336 {
 337     for (size_t u = 0; u < nkeywords; u++)
 338     {
 339         if (keywords[u].value == value)
 340             return 1;
 341     }
 342     return 0;
 343 }
 344
 345 struct TokenInitializer
 346 {
 347     TokenInitializer();
 348 };
 349
 350 static TokenInitializer tokeninitializer;
 351
 352 TokenInitializer::TokenInitializer()
 353 {
 354     Identifier::initTable();
 355     for (nkeywords = 0; keywords[nkeywords].name; nkeywords++)
 356     {
 357         //printf("keyword[%d] = '%s'\n",u, keywords[u].name);
 358         const char *s = keywords[nkeywords].name;
 359         size_t len = strlen(s);
 360         TOK v = keywords[nkeywords].value;
 361         Identifier::idPool(s, len, v);
 362
 363         //printf("tochars[%d] = '%s'\n",v, s);
 364         Token::tochars[v] = s;
 365     }
 366
 367     Token::tochars[TOKeof]              = "EOF";
 368     Token::tochars[TOKlcurly]           = "{";
 369     Token::tochars[TOKrcurly]           = "}";
 370     Token::tochars[TOKlparen]           = "(";
 371     Token::tochars[TOKrparen]           = ")";
 372     Token::tochars[TOKlbracket]         = "[";
 373     Token::tochars[TOKrbracket]         = "]";
 374     Token::tochars[TOKsemicolon]        = ";";
 375     Token::tochars[TOKcolon]            = ":";
 376     Token::tochars[TOKcomma]            = ",";
 377     Token::tochars[TOKdot]              = ".";
 378     Token::tochars[TOKxor]              = "^";
 379     Token::tochars[TOKxorass]           = "^=";
 380     Token::tochars[TOKassign]           = "=";
 381     Token::tochars[TOKconstruct]        = "=";
 382     Token::tochars[TOKblit]             = "=";
 383     Token::tochars[TOKlt]               = "<";
 384     Token::tochars[TOKgt]               = ">";
 385     Token::tochars[TOKle]               = "<=";
 386     Token::tochars[TOKge]               = ">=";
 387     Token::tochars[TOKequal]            = "==";
 388     Token::tochars[TOKnotequal]         = "!=";
 389     Token::tochars[TOKnotidentity]      = "!is";
 390
 391     Token::tochars[TOKunord]            = "!<>=";
 392     Token::tochars[TOKue]               = "!<>";
 393     Token::tochars[TOKlg]               = "<>";
 394     Token::tochars[TOKleg]              = "<>=";
 395     Token::tochars[TOKule]              = "!>";
 396     Token::tochars[TOKul]               = "!>=";
 397     Token::tochars[TOKuge]              = "!<";
 398     Token::tochars[TOKug]               = "!<=";
 399
 400     Token::tochars[TOKnot]              = "!";
 401     Token::tochars[TOKshl]              = "<<";
 402     Token::tochars[TOKshr]              = ">>";
 403     Token::tochars[TOKushr]             = ">>>";
 404     Token::tochars[TOKadd]              = "+";
 405     Token::tochars[TOKmin]              = "-";
 406     Token::tochars[TOKmul]              = "*";
 407     Token::tochars[TOKdiv]              = "/";
 408     Token::tochars[TOKmod]              = "%";
 409     Token::tochars[TOKslice]            = "..";
 410     Token::tochars[TOKdotdotdot]        = "...";
 411     Token::tochars[TOKand]              = "&";
 412     Token::tochars[TOKandand]           = "&&";
 413     Token::tochars[TOKor]               = "|";
 414     Token::tochars[TOKoror]             = "||";
 415     Token::tochars[TOKarray]            = "[]";
 416     Token::tochars[TOKindex]            = "[i]";
 417     Token::tochars[TOKaddress]          = "&";
 418     Token::tochars[TOKstar]             = "*";
 419     Token::tochars[TOKtilde]            = "~";
 420     Token::tochars[TOKdollar]           = "$";
 421     Token::tochars[TOKcast]             = "cast";
 422     Token::tochars[TOKplusplus]         = "++";
 423     Token::tochars[TOKminusminus]       = "--";
 424     Token::tochars[TOKpreplusplus]      = "++";
 425     Token::tochars[TOKpreminusminus]    = "--";
 426     Token::tochars[TOKtype]             = "type";
 427     Token::tochars[TOKquestion]         = "?";
 428     Token::tochars[TOKneg]              = "-";
 429     Token::tochars[TOKuadd]             = "+";
 430     Token::tochars[TOKvar]              = "var";
 431     Token::tochars[TOKaddass]           = "+=";
 432     Token::tochars[TOKminass]           = "-=";
 433     Token::tochars[TOKmulass]           = "*=";
 434     Token::tochars[TOKdivass]           = "/=";
 435     Token::tochars[TOKmodass]           = "%=";
 436     Token::tochars[TOKshlass]           = "<<=";
 437     Token::tochars[TOKshrass]           = ">>=";
 438     Token::tochars[TOKushrass]          = ">>>=";
 439     Token::tochars[TOKandass]           = "&=";
 440     Token::tochars[TOKorass]            = "|=";
 441     Token::tochars[TOKcatass]           = "~=";
 442     Token::tochars[TOKcat]              = "~";
 443     Token::tochars[TOKcall]             = "call";
 444     Token::tochars[TOKidentity]         = "is";
 445     Token::tochars[TOKnotidentity]      = "!is";
 446
 447     Token::tochars[TOKorass]            = "|=";
 448     Token::tochars[TOKidentifier]       = "identifier";
 449     Token::tochars[TOKat]               = "@";
 450     Token::tochars[TOKpow]              = "^^";
 451     Token::tochars[TOKpowass]           = "^^=";
 452     Token::tochars[TOKgoesto]           = "=>";
 453     Token::tochars[TOKpound]            = "#";
 454
 455      // For debugging
 456     Token::tochars[TOKerror]            = "error";
 457     Token::tochars[TOKdotid]            = "dotid";
 458     Token::tochars[TOKdottd]            = "dottd";
 459     Token::tochars[TOKdotti]            = "dotti";
 460     Token::tochars[TOKdotvar]           = "dotvar";
 461     Token::tochars[TOKdottype]          = "dottype";
 462     Token::tochars[TOKsymoff]           = "symoff";
 463     Token::tochars[TOKarraylength]      = "arraylength";
 464     Token::tochars[TOKarrayliteral]     = "arrayliteral";
 465     Token::tochars[TOKassocarrayliteral] = "assocarrayliteral";
 466     Token::tochars[TOKstructliteral]    = "structliteral";
 467     Token::tochars[TOKstring]           = "string";
 468     Token::tochars[TOKdsymbol]          = "symbol";
 469     Token::tochars[TOKtuple]            = "tuple";
 470     Token::tochars[TOKdeclaration]      = "declaration";
 471     Token::tochars[TOKon_scope_exit]    = "scope(exit)";
 472     Token::tochars[TOKon_scope_success] = "scope(success)";
 473     Token::tochars[TOKon_scope_failure] = "scope(failure)";
 474     Token::tochars[TOKdelegateptr]      = "delegateptr";
 475     Token::tochars[TOKvectorarray]      = "vectorarray";
 476 }