2 * Defines lexical tokens.
4 * Specification: $(LINK2 https://dlang.org/spec/lex.html#tokens, Tokens)
6 * Copyright: Copyright (C) 1999-2021 by The D Language Foundation, All Rights Reserved
7 * Authors: $(LINK2 http://www.digitalmars.com, Walter Bright)
8 * License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt, Boost License 1.0)
9 * Source: $(LINK2 https://github.com/dlang/dmd/blob/master/src/dmd/tokens.d, _tokens.d)
10 * Documentation: https://dlang.org/phobos/dmd_tokens.html
11 * Coverage: https://codecov.io/gh/dlang/dmd/src/master/src/dmd/tokens.d
16 import core.stdc.ctype;
17 import core.stdc.stdio;
18 import core.stdc.string;
20 import dmd.identifier;
21 import dmd.root.ctfloat;
22 import dmd.common.outbuffer;
67 dotTemplateDeclaration,
83 delegateFunctionPointer,
102 unsignedRightShiftAssign,
104 concatenateAssign, // ~=
105 concatenateElemAssign,
106 concatenateDcharAssign,
271 moduleString, // __MODULE__
272 functionString, // __FUNCTION__
273 prettyFunction, // __PRETTY_FUNCTION__
293 compoundLiteral, // ( type-name ) { initializer-list }
315 // C only extended keywords
352 dotTemplateDeclaration,
367 delegateFunctionPointer,
386 unsignedRightShiftAssign,
388 concatenateAssign, // ~=
389 concatenateElemAssign,
390 concatenateDcharAssign,
453 moduleString, // __MODULE__
454 functionString, // __FUNCTION__
455 prettyFunction, // __PRETTY_FUNCTION__
467 compoundLiteral, // ( type-name ) { initializer-list }
472 enum FirstCKeyword = TOK.inline;
474 // Assert that all token enum members have consecutive values and
475 // that none of them overlap
477 foreach (idx, enumName; __traits(allMembers, TOK)) {
478 static if (idx != __traits(getMember, TOK, enumName)) {
479 pragma(msg, "Error: Expected TOK.", enumName, " to be ", idx, " but is ", __traits(getMember, TOK, enumName));
486 /****************************************
489 private immutable TOK[] keywords =
552 TOK.foreach_reverse_,
623 // C only extended keywords
629 // Initialize the identifier pool
630 shared static this() nothrow
632 Identifier.initTable();
633 foreach (kw; keywords)
635 //printf("keyword[%d] = '%s'\n",kw, tochars[kw].ptr);
636 Identifier.idPool(Token.tochars[kw].ptr, Token.tochars[kw].length, cast(uint)kw);
640 /************************************
641 * This is used to pick the C keywords out of the tokens.
642 * If it's not a C keyword, then it's an identifier.
644 static immutable TOK[TOK.max + 1] Ckeywords =
648 TOK[TOK.max + 1] tab = identifier; // default to identifier
649 enum Ckwds = [ auto_, break_, case_, char_, const_, continue_, default_, do_, float64, else_,
650 enum_, extern_, float32, for_, goto_, if_, inline, int32, int64, register,
651 restrict, return_, int16, signed, sizeof_, static_, struct_, switch_, typedef_,
652 union_, unsigned, void_, volatile, while_, asm_,
653 _Alignas, _Alignof, _Atomic, _Bool, _Complex, _Generic, _Imaginary, _Noreturn,
654 _Static_assert, _Thread_local, __cdecl, __declspec, __attribute__ ];
657 tab[kw] = cast(TOK) kw;
664 /***********************************************************
666 extern (C++) struct Token
670 const(char)* ptr; // pointer to first character of this token within buffer
672 const(char)[] blockComment; // doc comment string prior to this token
673 const(char)[] lineComment; // doc comment for previous token
685 const(char)* ustring; // UTF8 string
687 ubyte postfix; // 'c', 'w', 'd'
693 extern (D) private static immutable string[TOK.max + 1] tochars =
698 TOK.assert_: "assert",
704 TOK.delete_: "delete",
706 TOK.module_: "module",
707 TOK.pragma_: "pragma",
708 TOK.typeof_: "typeof",
709 TOK.typeid_: "typeid",
710 TOK.template_: "template",
722 TOK.float32: "float",
723 TOK.float64: "double",
729 TOK.imaginary32: "ifloat",
730 TOK.imaginary64: "idouble",
731 TOK.imaginary80: "ireal",
732 TOK.complex32: "cfloat",
733 TOK.complex64: "cdouble",
734 TOK.complex80: "creal",
735 TOK.delegate_: "delegate",
736 TOK.function_: "function",
743 TOK.switch_: "switch",
745 TOK.default_: "default",
747 TOK.continue_: "continue",
748 TOK.synchronized_: "synchronized",
749 TOK.return_: "return",
753 TOK.finally_: "finally",
756 TOK.foreach_: "foreach",
757 TOK.foreach_reverse_: "foreach_reverse",
759 TOK.struct_: "struct",
761 TOK.interface_: "interface",
764 TOK.import_: "import",
766 TOK.static_: "static",
770 TOK.override_: "override",
771 TOK.abstract_: "abstract",
773 TOK.deprecated_: "deprecated",
780 TOK.extern_: "extern",
781 TOK.private_: "private",
782 TOK.package_: "package",
783 TOK.protected_: "protected",
784 TOK.public_: "public",
785 TOK.export_: "export",
786 TOK.invariant_: "invariant",
787 TOK.unittest_: "unittest",
788 TOK.version_: "version",
789 TOK.argumentTypes: "__argTypes",
790 TOK.parameters: "__parameters",
794 TOK.nothrow_: "nothrow",
795 TOK.gshared: "__gshared",
796 TOK.traits: "__traits",
797 TOK.vector: "__vector",
798 TOK.overloadSet: "__overloadset",
799 TOK.file: "__FILE__",
800 TOK.fileFullPath: "__FILE_FULL_PATH__",
801 TOK.line: "__LINE__",
802 TOK.moduleString: "__MODULE__",
803 TOK.functionString: "__FUNCTION__",
804 TOK.prettyFunction: "__PRETTY_FUNCTION__",
805 TOK.shared_: "shared",
806 TOK.immutable_: "immutable",
808 TOK.endOfFile: "End of File",
811 TOK.leftParenthesis: "(",
812 TOK.rightParenthesis: ")",
813 TOK.leftBracket: "[",
814 TOK.rightBracket: "]",
825 TOK.greaterThan: ">",
826 TOK.lessOrEqual: "<=",
827 TOK.greaterOrEqual: ">=",
832 TOK.rightShift: ">>",
833 TOK.unsignedRightShift: ">>>",
840 TOK.dotDotDot: "...",
852 TOK.minusMinus: "--",
853 TOK.prePlusPlus: "++",
854 TOK.preMinusMinus: "--",
865 TOK.leftShiftAssign: "<<=",
866 TOK.rightShiftAssign: ">>=",
867 TOK.unsignedRightShiftAssign: ">>>=",
870 TOK.concatenateAssign: "~=",
871 TOK.concatenateElemAssign: "~=",
872 TOK.concatenateDcharAssign: "~=",
873 TOK.concatenate: "~",
876 TOK.notIdentity: "!is",
877 TOK.identifier: "identifier",
880 TOK.powAssign: "^^=",
884 TOK.colonColon: "::",
888 TOK.dotIdentifier: "dotid",
889 TOK.dotTemplateDeclaration: "dottd",
890 TOK.dotTemplateInstance: "dotti",
891 TOK.dotVariable: "dotvar",
892 TOK.dotType: "dottype",
893 TOK.symbolOffset: "symoff",
894 TOK.arrayLength: "arraylength",
895 TOK.arrayLiteral: "arrayliteral",
896 TOK.assocArrayLiteral: "assocarrayliteral",
897 TOK.structLiteral: "structliteral",
898 TOK.string_: "string",
899 TOK.dSymbol: "symbol",
901 TOK.declaration: "declaration",
902 TOK.onScopeExit: "scope(exit)",
903 TOK.onScopeSuccess: "scope(success)",
904 TOK.onScopeFailure: "scope(failure)",
905 TOK.delegatePointer: "delegateptr",
908 TOK.reserved: "reserved",
909 TOK.remove: "remove",
910 TOK.newAnonymousClass: "newanonclass",
911 TOK.comment: "comment",
912 TOK.classReference: "classreference",
913 TOK.thrownException: "thrownexception",
914 TOK.delegateFunctionPointer: "delegatefuncptr",
915 TOK.int32Literal: "int32v",
916 TOK.uns32Literal: "uns32v",
917 TOK.int64Literal: "int64v",
918 TOK.uns64Literal: "uns64v",
919 TOK.int128Literal: "int128v",
920 TOK.uns128Literal: "uns128v",
921 TOK.float32Literal: "float32v",
922 TOK.float64Literal: "float64v",
923 TOK.float80Literal: "float80v",
924 TOK.imaginary32Literal: "imaginary32v",
925 TOK.imaginary64Literal: "imaginary64v",
926 TOK.imaginary80Literal: "imaginary80v",
927 TOK.charLiteral: "charv",
928 TOK.wcharLiteral: "wcharv",
929 TOK.dcharLiteral: "dcharv",
930 TOK.wchar_tLiteral: "wchar_tv",
931 TOK.compoundLiteral: "compoundliteral",
934 TOK.hexadecimalString: "xstring",
936 TOK.interval: "interval",
937 TOK.voidExpression: "voidexp",
938 TOK.cantExpression: "cantexp",
939 TOK.showCtfeContext : "showCtfeContext",
941 TOK.objcClassReference: "class",
942 TOK.vectorArray: "vectorarray",
945 TOK.inline : "inline",
946 TOK.register : "register",
947 TOK.restrict : "restrict",
948 TOK.signed : "signed",
949 TOK.sizeof_ : "sizeof",
950 TOK.typedef_ : "typedef",
951 TOK.unsigned : "unsigned",
952 TOK.volatile : "volatile",
953 TOK._Alignas : "_Alignas",
954 TOK._Alignof : "_Alignof",
955 TOK._Atomic : "_Atomic",
957 TOK._Complex : "_Complex",
958 TOK._Generic : "_Generic",
959 TOK._Imaginary: "_Imaginary",
960 TOK._Noreturn : "_Noreturn",
961 TOK._Static_assert : "_Static_assert",
962 TOK._Thread_local : "_Thread_local",
964 // C only extended keywords
965 TOK.__cdecl : "__cdecl",
966 TOK.__declspec : "__declspec",
967 TOK.__attribute__ : "__attribute__",
978 int isKeyword() const
980 foreach (kw; keywords)
989 * Set to contents of ptr[0..length]
991 * ptr = pointer to string
992 * length = length of string
994 void setString(const(char)* ptr, size_t length)
996 auto s = cast(char*)mem.xmalloc_noscan(length + 1);
997 memcpy(s, ptr, length);
1000 len = cast(uint)length;
1005 * Set to contents of buf
1007 * buf = string (not zero terminated)
1009 void setString(const ref OutBuffer buf)
1011 setString(cast(const(char)*)buf[].ptr, buf.length);
1015 * Set to empty string
1024 extern (C++) const(char)* toChars() const
1026 __gshared char[3 + 3 * floatvalue.sizeof + 1] buffer;
1027 const(char)* p = &buffer[0];
1030 case TOK.int32Literal:
1031 sprintf(&buffer[0], "%d", cast(d_int32)intvalue);
1033 case TOK.uns32Literal:
1034 case TOK.charLiteral:
1035 case TOK.wcharLiteral:
1036 case TOK.dcharLiteral:
1037 case TOK.wchar_tLiteral:
1038 sprintf(&buffer[0], "%uU", cast(d_uns32)unsvalue);
1040 case TOK.int64Literal:
1041 sprintf(&buffer[0], "%lldL", cast(long)intvalue);
1043 case TOK.uns64Literal:
1044 sprintf(&buffer[0], "%lluUL", cast(ulong)unsvalue);
1046 case TOK.float32Literal:
1047 CTFloat.sprint(&buffer[0], 'g', floatvalue);
1048 strcat(&buffer[0], "f");
1050 case TOK.float64Literal:
1051 CTFloat.sprint(&buffer[0], 'g', floatvalue);
1053 case TOK.float80Literal:
1054 CTFloat.sprint(&buffer[0], 'g', floatvalue);
1055 strcat(&buffer[0], "L");
1057 case TOK.imaginary32Literal:
1058 CTFloat.sprint(&buffer[0], 'g', floatvalue);
1059 strcat(&buffer[0], "fi");
1061 case TOK.imaginary64Literal:
1062 CTFloat.sprint(&buffer[0], 'g', floatvalue);
1063 strcat(&buffer[0], "i");
1065 case TOK.imaginary80Literal:
1066 CTFloat.sprint(&buffer[0], 'g', floatvalue);
1067 strcat(&buffer[0], "Li");
1073 for (size_t i = 0; i < len;)
1076 utf_decodeChar(ustring[0 .. len], i, c);
1083 buf.writeByte('\\');
1091 buf.printf("\\x%02x", c);
1093 else if (c <= 0xFFFF)
1094 buf.printf("\\u%04x", c);
1096 buf.printf("\\U%08x", c);
1103 buf.writeByte(postfix);
1105 p = buf.extractSlice().ptr;
1108 case TOK.hexadecimalString:
1113 foreach (size_t i; 0 .. len)
1117 buf.printf("%02x", ustring[i]);
1121 buf.writeByte(postfix);
1123 p = buf.extractSlice().ptr;
1126 case TOK.identifier:
1147 case TOK.imaginary32:
1148 case TOK.imaginary64:
1149 case TOK.imaginary80:
1154 p = ident.toChars();
1163 static const(char)* toChars(TOK value)
1165 return toString(value).ptr;
1168 static const(char)* toChars(ushort value)
1170 return toString(cast(TOK)value).ptr;
1173 extern (D) static string toString(TOK value) pure nothrow @nogc @safe
1175 return tochars[value];