gcc/d/dmd/dmodule.d

   1 /**
   2  * Defines a package and module.
   3  *
   4  * Specification: $(LINK2 https://dlang.org/spec/module.html, Modules)
   5  *
   6  * Copyright:   Copyright (C) 1999-2024 by The D Language Foundation, All Rights Reserved
   7  * Authors:     $(LINK2 https://www.digitalmars.com, Walter Bright)
   8  * License:     $(LINK2 https://www.boost.org/LICENSE_1_0.txt, Boost License 1.0)
   9  * Source:      $(LINK2 https://github.com/dlang/dmd/blob/master/src/dmd/dmodule.d, _dmodule.d)
  10  * Documentation:  https://dlang.org/phobos/dmd_dmodule.html
  11  * Coverage:    https://codecov.io/gh/dlang/dmd/src/master/src/dmd/dmodule.d
  12  */
  13
  14 module dmd.dmodule;
  15
  16 import core.stdc.stdio;
  17 import core.stdc.stdlib;
  18 import core.stdc.string;
  19
  20 import dmd.aggregate;
  21 import dmd.arraytypes;
  22 import dmd.astcodegen;
  23 import dmd.astenums;
  24 import dmd.common.outbuffer;
  25 import dmd.compiler;
  26 import dmd.cparse;
  27 import dmd.dimport;
  28 import dmd.dmacro;
  29 import dmd.doc;
  30 import dmd.dscope;
  31 import dmd.dsymbol;
  32 import dmd.dsymbolsem;
  33 import dmd.errors;
  34 import dmd.errorsink;
  35 import dmd.expression;
  36 import dmd.expressionsem;
  37 import dmd.file_manager;
  38 import dmd.func;
  39 import dmd.globals;
  40 import dmd.gluelayer;
  41 import dmd.id;
  42 import dmd.identifier;
  43 import dmd.location;
  44 import dmd.parse;
  45 import dmd.root.array;
  46 import dmd.root.file;
  47 import dmd.root.filename;
  48 import dmd.root.port;
  49 import dmd.root.rmem;
  50 import dmd.root.string;
  51 import dmd.rootobject;
  52 import dmd.semantic2;
  53 import dmd.semantic3;
  54 import dmd.target;
  55 import dmd.utils;
  56 import dmd.visitor;
  57
  58 version (Windows)
  59 {
  60     import core.sys.windows.winbase : getpid = GetCurrentProcessId;
  61     enum PathSeparator = '\\';
  62 }
  63 else version (Posix)
  64 {
  65     import core.sys.posix.unistd : getpid;
  66     enum PathSeparator = '/';
  67 }
  68 else
  69     static assert(0);
  70
  71 version (IN_GCC) {}
  72 else version (IN_LLVM) {}
  73 else version = MARS;
  74
  75 // function used to call semantic3 on a module's dependencies
  76 void semantic3OnDependencies(Module m)
  77 {
  78     if (!m)
  79         return;
  80
  81     if (m.semanticRun > PASS.semantic3)
  82         return;
  83
  84     m.semantic3(null);
  85
  86     foreach (i; 1 .. m.aimports.length)
  87         semantic3OnDependencies(m.aimports[i]);
  88 }
  89
  90 /**
  91  * Remove generated .di files on error and exit
  92  */
  93 void removeHdrFilesAndFail(ref Param params, ref Modules modules) nothrow
  94 {
  95     if (params.dihdr.doOutput)
  96     {
  97         foreach (m; modules)
  98         {
  99             if (m.filetype == FileType.dhdr)
 100                 continue;
 101             File.remove(m.hdrfile.toChars());
 102         }
 103     }
 104
 105     fatal();
 106 }
 107
 108 /**
 109  * Converts a chain of identifiers to the filename of the module
 110  *
 111  * Params:
 112  *  packages = the names of the "parent" packages
 113  *  ident = the name of the child package or module
 114  *
 115  * Returns:
 116  *  the filename of the child package or module
 117  */
 118 private const(char)[] getFilename(Identifier[] packages, Identifier ident) nothrow
 119 {
 120     const(char)[] filename = ident.toString();
 121
 122     OutBuffer buf;
 123     OutBuffer dotmods;
 124     auto modAliases = &global.params.modFileAliasStrings;
 125
 126     if (packages.length == 0 && modAliases.length == 0)
 127         return filename;
 128
 129     void checkModFileAlias(const(char)[] p)
 130     {
 131         /* Check and replace the contents of buf[] with
 132         * an alias string from global.params.modFileAliasStrings[]
 133         */
 134         dotmods.writestring(p);
 135         foreach_reverse (const m; *modAliases)
 136         {
 137             const q = strchr(m, '=');
 138             assert(q);
 139             if (dotmods.length == q - m && memcmp(dotmods.peekChars(), m, q - m) == 0)
 140             {
 141                 buf.setsize(0);
 142                 auto rhs = q[1 .. strlen(q)];
 143                 if (rhs.length > 0 && (rhs[$ - 1] == '/' || rhs[$ - 1] == '\\'))
 144                     rhs = rhs[0 .. $ - 1]; // remove trailing separator
 145                 buf.writestring(rhs);
 146                 break; // last matching entry in ms[] wins
 147             }
 148         }
 149         dotmods.writeByte('.');
 150     }
 151
 152     foreach (pid; packages)
 153     {
 154         const p = pid.toString();
 155         buf.writestring(p);
 156         if (modAliases.length)
 157             checkModFileAlias(p);
 158         buf.writeByte(PathSeparator);
 159     }
 160     buf.writestring(filename);
 161     if (modAliases.length)
 162         checkModFileAlias(filename);
 163     buf.writeByte(0);
 164     filename = buf.extractSlice()[0 .. $ - 1];
 165
 166     return filename;
 167 }
 168
 169 /***********************************************************
 170  */
 171 extern (C++) class Package : ScopeDsymbol
 172 {
 173     PKG isPkgMod = PKG.unknown;
 174     uint tag;        // auto incremented tag, used to mask package tree in scopes
 175     Module mod;     // !=null if isPkgMod == PKG.module_
 176
 177     final extern (D) this(const ref Loc loc, Identifier ident) nothrow
 178     {
 179         super(loc, ident);
 180         __gshared uint packageTag;
 181         this.tag = packageTag++;
 182     }
 183
 184     override const(char)* kind() const nothrow
 185     {
 186         return "package";
 187     }
 188
 189     override bool equals(const RootObject o) const
 190     {
 191         // custom 'equals' for bug 17441. "package a" and "module a" are not equal
 192         if (this == o)
 193             return true;
 194         auto p = cast(Package)o;
 195         return p && isModule() == p.isModule() && ident.equals(p.ident);
 196     }
 197
 198     /****************************************************
 199      * Input:
 200      *      packages[]      the pkg1.pkg2 of pkg1.pkg2.mod
 201      * Returns:
 202      *      the symbol table that mod should be inserted into
 203      * Output:
 204      *      *pparent        the rightmost package, i.e. pkg2, or NULL if no packages
 205      *      *ppkg           the leftmost package, i.e. pkg1, or NULL if no packages
 206      */
 207     extern (D) static DsymbolTable resolve(Identifier[] packages, Dsymbol* pparent, Package* ppkg)
 208     {
 209         DsymbolTable dst = Module.modules;
 210         Dsymbol parent = null;
 211         //printf("Package::resolve()\n");
 212         if (ppkg)
 213             *ppkg = null;
 214         foreach (pid; packages)
 215         {
 216             Package pkg;
 217             Dsymbol p = dst.lookup(pid);
 218             if (!p)
 219             {
 220                 pkg = new Package(Loc.initial, pid);
 221                 dst.insert(pkg);
 222                 pkg.parent = parent;
 223                 pkg.symtab = new DsymbolTable();
 224             }
 225             else
 226             {
 227                 pkg = p.isPackage();
 228                 assert(pkg);
 229                 // It might already be a module, not a package, but that needs
 230                 // to be checked at a higher level, where a nice error message
 231                 // can be generated.
 232                 // dot net needs modules and packages with same name
 233                 // But we still need a symbol table for it
 234                 if (!pkg.symtab)
 235                     pkg.symtab = new DsymbolTable();
 236             }
 237             parent = pkg;
 238             dst = pkg.symtab;
 239             if (ppkg && !*ppkg)
 240                 *ppkg = pkg;
 241             if (pkg.isModule())
 242             {
 243                 // Return the module so that a nice error message can be generated
 244                 if (ppkg)
 245                     *ppkg = cast(Package)p;
 246                 break;
 247             }
 248         }
 249
 250         if (pparent)
 251             *pparent = parent;
 252         return dst;
 253     }
 254
 255     override final inout(Package) isPackage() inout
 256     {
 257         return this;
 258     }
 259
 260     /**
 261      * Checks if pkg is a sub-package of this
 262      *
 263      * For example, if this qualifies to 'a1.a2' and pkg - to 'a1.a2.a3',
 264      * this function returns 'true'. If it is other way around or qualified
 265      * package paths conflict function returns 'false'.
 266      *
 267      * Params:
 268      *  pkg = possible subpackage
 269      *
 270      * Returns:
 271      *  see description
 272      */
 273     final bool isAncestorPackageOf(const Package pkg) const
 274     {
 275         if (this == pkg)
 276             return true;
 277         if (!pkg || !pkg.parent)
 278             return false;
 279         return isAncestorPackageOf(pkg.parent.isPackage());
 280     }
 281
 282     override void accept(Visitor v)
 283     {
 284         v.visit(this);
 285     }
 286
 287     final Module isPackageMod()
 288     {
 289         if (isPkgMod == PKG.module_)
 290         {
 291             return mod;
 292         }
 293         return null;
 294     }
 295
 296     /**
 297      * Checks for the existence of a package.d to set isPkgMod appropriately
 298      * if isPkgMod == PKG.unknown
 299      */
 300     final void resolvePKGunknown()
 301     {
 302         if (isModule())
 303             return;
 304         if (isPkgMod != PKG.unknown)
 305             return;
 306
 307         Identifier[] packages;
 308         for (Dsymbol s = this.parent; s; s = s.parent)
 309             packages ~= s.ident;
 310         reverse(packages);
 311
 312         if (Module.find(getFilename(packages, ident)))
 313             Module.load(Loc.initial, packages, this.ident);
 314         else
 315             isPkgMod = PKG.package_;
 316     }
 317 }
 318
 319 /***********************************************************
 320  */
 321 extern (C++) final class Module : Package
 322 {
 323     extern (C++) __gshared Module rootModule;
 324     extern (C++) __gshared DsymbolTable modules; // symbol table of all modules
 325     extern (C++) __gshared Modules amodules;     // array of all modules
 326     extern (C++) __gshared Dsymbols deferred;    // deferred Dsymbol's needing semantic() run on them
 327     extern (C++) __gshared Dsymbols deferred2;   // deferred Dsymbol's needing semantic2() run on them
 328     extern (C++) __gshared Dsymbols deferred3;   // deferred Dsymbol's needing semantic3() run on them
 329
 330     static void _init()
 331     {
 332         modules = new DsymbolTable();
 333     }
 334
 335     /**
 336      * Deinitializes the global state of the compiler.
 337      *
 338      * This can be used to restore the state set by `_init` to its original
 339      * state.
 340      */
 341     static void deinitialize()
 342     {
 343         modules = modules.init;
 344     }
 345
 346     extern (C++) __gshared AggregateDeclaration moduleinfo;
 347
 348     const(char)[] arg;           // original argument name
 349     ModuleDeclaration* md;      // if !=null, the contents of the ModuleDeclaration declaration
 350     const FileName srcfile;     // input source file
 351     const FileName objfile;     // output .obj file
 352     const FileName hdrfile;     // 'header' file
 353     FileName docfile;           // output documentation file
 354     const(ubyte)[] src;         /// Raw content of the file
 355     uint errors;                // if any errors in file
 356     uint numlines;              // number of lines in source file
 357     FileType filetype;          // source file type
 358     bool hasAlwaysInlines;      // contains references to functions that must be inlined
 359     bool isPackageFile;         // if it is a package.d
 360     Package pkg;                // if isPackageFile is true, the Package that contains this package.d
 361     Strings contentImportedFiles; // array of files whose content was imported
 362     int needmoduleinfo;
 363     private ThreeState selfimports;
 364     private ThreeState rootimports;
 365     Dsymbol[void*] tagSymTab;   /// ImportC: tag symbols that conflict with other symbols used as the index
 366
 367     private OutBuffer defines;  // collect all the #define lines here
 368
 369
 370     /*************************************
 371      * Return true if module imports itself.
 372      */
 373     bool selfImports()
 374     {
 375         //printf("Module::selfImports() %s\n", toChars());
 376         if (selfimports == ThreeState.none)
 377         {
 378             foreach (Module m; amodules)
 379                 m.insearch = false;
 380             selfimports = imports(this) ? ThreeState.yes : ThreeState.no;
 381             foreach (Module m; amodules)
 382                 m.insearch = false;
 383         }
 384         return selfimports == ThreeState.yes;
 385     }
 386
 387     /*************************************
 388      * Return true if module imports root module.
 389      */
 390     bool rootImports()
 391     {
 392         //printf("Module::rootImports() %s\n", toChars());
 393         if (rootimports == ThreeState.none)
 394         {
 395             foreach (Module m; amodules)
 396                 m.insearch = false;
 397             rootimports = ThreeState.no;
 398             foreach (Module m; amodules)
 399             {
 400                 if (m.isRoot() && imports(m))
 401                 {
 402                     rootimports = ThreeState.yes;
 403                     break;
 404                 }
 405             }
 406             foreach (Module m; amodules)
 407                 m.insearch = false;
 408         }
 409         return rootimports == ThreeState.yes;
 410     }
 411
 412     Identifier searchCacheIdent;
 413     Dsymbol searchCacheSymbol;  // cached value of search
 414     SearchOptFlags searchCacheFlags;       // cached flags
 415     bool insearch;
 416
 417     /**
 418      * A root module is one that will be compiled all the way to
 419      * object code.  This field holds the root module that caused
 420      * this module to be loaded.  If this module is a root module,
 421      * then it will be set to `this`.  This is used to determine
 422      * ownership of template instantiation.
 423      */
 424     Module importedFrom;
 425
 426     Dsymbols* decldefs;         // top level declarations for this Module
 427
 428     Modules aimports;           // all imported modules
 429
 430     uint debuglevel;            // debug level
 431     Identifiers* debugids;      // debug identifiers
 432     Identifiers* debugidsNot;   // forward referenced debug identifiers
 433
 434     uint versionlevel;          // version level
 435     Identifiers* versionids;    // version identifiers
 436     Identifiers* versionidsNot; // forward referenced version identifiers
 437
 438     MacroTable macrotable;      // document comment macros
 439     Escape* _escapetable;       // document comment escapes
 440
 441     size_t nameoffset;          // offset of module name from start of ModuleInfo
 442     size_t namelen;             // length of module name in characters
 443
 444     extern (D) this(const ref Loc loc, const(char)[] filename, Identifier ident, int doDocComment, int doHdrGen)
 445     {
 446         super(loc, ident);
 447         const(char)[] srcfilename;
 448         //printf("Module::Module(filename = '%.*s', ident = '%s')\n", cast(int)filename.length, filename.ptr, ident.toChars());
 449         this.arg = filename;
 450         srcfilename = FileName.defaultExt(filename, mars_ext);
 451         if (target.run_noext && global.params.run &&
 452             !FileName.ext(filename) &&
 453             FileName.exists(srcfilename) == 0 &&
 454             FileName.exists(filename) == 1)
 455         {
 456             FileName.free(srcfilename.ptr);
 457             srcfilename = FileName.removeExt(filename); // just does a mem.strdup(filename)
 458         }
 459         else if (!FileName.equalsExt(srcfilename, mars_ext) &&
 460                  !FileName.equalsExt(srcfilename, hdr_ext) &&
 461                  !FileName.equalsExt(srcfilename, c_ext) &&
 462                  !FileName.equalsExt(srcfilename, i_ext) &&
 463                  !FileName.equalsExt(srcfilename, dd_ext))
 464         {
 465
 466             error(loc, "%s `%s` source file name '%.*s' must have .%.*s extension",
 467                   kind, toPrettyChars,
 468                   cast(int)srcfilename.length, srcfilename.ptr,
 469                   cast(int)mars_ext.length, mars_ext.ptr);
 470             fatal();
 471         }
 472
 473         srcfile = FileName(srcfilename);
 474         objfile = setOutfilename(global.params.objname, global.params.objdir, filename, target.obj_ext);
 475         if (doDocComment)
 476             setDocfile();
 477         if (doHdrGen)
 478             hdrfile = setOutfilename(global.params.dihdr.name, global.params.dihdr.dir, arg, hdr_ext);
 479     }
 480
 481     extern (D) this(const(char)[] filename, Identifier ident, int doDocComment, int doHdrGen)
 482     {
 483         this(Loc.initial, filename, ident, doDocComment, doHdrGen);
 484     }
 485
 486     static Module create(const(char)* filename, Identifier ident, int doDocComment, int doHdrGen)
 487     {
 488         return create(filename.toDString, ident, doDocComment, doHdrGen);
 489     }
 490
 491     extern (D) static Module create(const(char)[] filename, Identifier ident, int doDocComment, int doHdrGen)
 492     {
 493         return new Module(Loc.initial, filename, ident, doDocComment, doHdrGen);
 494     }
 495
 496     static const(char)* find(const(char)* filename)
 497     {
 498         return find(filename.toDString).ptr;
 499     }
 500
 501     extern (D) static const(char)[] find(const(char)[] filename)
 502     {
 503         return global.fileManager.lookForSourceFile(filename, global.path[]);
 504     }
 505
 506     extern (C++) static Module load(const ref Loc loc, Identifiers* packages, Identifier ident)
 507     {
 508         return load(loc, packages ? (*packages)[] : null, ident);
 509     }
 510
 511     extern (D) static Module load(const ref Loc loc, Identifier[] packages, Identifier ident)
 512     {
 513         //printf("Module::load(ident = '%s')\n", ident.toChars());
 514         // Build module filename by turning:
 515         //  foo.bar.baz
 516         // into:
 517         //  foo\bar\baz
 518         const(char)[] filename = getFilename(packages, ident);
 519         // Look for the source file
 520         if (const result = find(filename))
 521             filename = result; // leaks
 522
 523         auto m = new Module(loc, filename, ident, 0, 0);
 524
 525         if (!m.read(loc))
 526             return null;
 527         if (global.params.v.verbose)
 528         {
 529             OutBuffer buf;
 530             foreach (pid; packages)
 531             {
 532                 buf.writestring(pid.toString());
 533                 buf.writeByte('.');
 534             }
 535             buf.printf("%s\t(%s)", ident.toChars(), m.srcfile.toChars());
 536             message("import    %s", buf.peekChars());
 537         }
 538         if((m = m.parse()) is null) return null;
 539
 540         return m;
 541     }
 542
 543     override const(char)* kind() const
 544     {
 545         return "module";
 546     }
 547
 548     /*********************************************
 549      * Combines things into output file name for .html and .di files.
 550      * Input:
 551      *      name    Command line name given for the file, NULL if none
 552      *      dir     Command line directory given for the file, NULL if none
 553      *      arg     Name of the source file
 554      *      ext     File name extension to use if 'name' is NULL
 555      *      global.params.preservePaths     get output path from arg
 556      *      srcfile Input file - output file name must not match input file
 557      */
 558     extern(D) FileName setOutfilename(const(char)[] name, const(char)[] dir, const(char)[] arg, const(char)[] ext)
 559     {
 560         const(char)[] docfilename;
 561         if (name)
 562         {
 563             docfilename = name;
 564         }
 565         else
 566         {
 567             const(char)[] argdoc;
 568             OutBuffer buf;
 569             if (arg == "__stdin.d")
 570             {
 571                 buf.printf("__stdin_%d.d", getpid());
 572                 arg = buf[];
 573             }
 574             if (global.params.preservePaths)
 575                 argdoc = arg;
 576             else
 577                 argdoc = FileName.name(arg);
 578             // If argdoc doesn't have an absolute path, make it relative to dir
 579             if (!FileName.absolute(argdoc))
 580             {
 581                 //FileName::ensurePathExists(dir);
 582                 argdoc = FileName.combine(dir, argdoc);
 583             }
 584             docfilename = FileName.forceExt(argdoc, ext);
 585         }
 586         if (FileName.equals(docfilename, srcfile.toString()))
 587         {
 588             error(loc, "%s `%s` source file and output file have same name '%s'",
 589                 kind, toPrettyChars, srcfile.toChars());
 590             fatal();
 591         }
 592         return FileName(docfilename);
 593     }
 594
 595     extern (D) void setDocfile()
 596     {
 597         docfile = setOutfilename(global.params.ddoc.name, global.params.ddoc.dir, arg, doc_ext);
 598     }
 599
 600     /**
 601      * Trigger the relevant semantic error when a file cannot be read
 602      *
 603      * We special case `object.d` as a failure is likely to be a rare
 604      * but difficult to diagnose case for the user. Packages also require
 605      * special handling to avoid exposing the compiler's internals.
 606      *
 607      * Params:
 608      *  loc = The location at which the file read originated (e.g. import)
 609      */
 610     private void onFileReadError(const ref Loc loc)
 611     {
 612         if (FileName.equals(srcfile.toString(), "object.d"))
 613         {
 614             .error(loc, "cannot find source code for runtime library file 'object.d'");
 615             version (IN_LLVM)
 616             {
 617                 errorSupplemental(loc, "ldc2 might not be correctly installed.");
 618                 errorSupplemental(loc, "Please check your ldc2.conf configuration file.");
 619                 errorSupplemental(loc, "Installation instructions can be found at http://wiki.dlang.org/LDC.");
 620             }
 621             version (MARS)
 622             {
 623                 errorSupplemental(loc, "dmd might not be correctly installed. Run 'dmd -man' for installation instructions.");
 624                 const dmdConfFile = global.inifilename.length ? FileName.canonicalName(global.inifilename) : "not found";
 625                 errorSupplemental(loc, "config file: %.*s", cast(int)dmdConfFile.length, dmdConfFile.ptr);
 626             }
 627         }
 628         else if (FileName.ext(this.arg) || !loc.isValid())
 629         {
 630             // Modules whose original argument name has an extension, or do not
 631             // have a valid location come from the command-line.
 632             // Error that their file cannot be found and return early.
 633             .error(loc, "cannot find input file `%s`", srcfile.toChars());
 634         }
 635         else
 636         {
 637             // if module is not named 'package' but we're trying to read 'package.d', we're looking for a package module
 638             bool isPackageMod = (strcmp(toChars(), "package") != 0) && isPackageFileName(srcfile);
 639             if (isPackageMod)
 640                 .error(loc, "importing package '%s' requires a 'package.d' file which cannot be found in '%s'", toChars(), srcfile.toChars());
 641             else
 642             {
 643                 .error(loc, "unable to read module `%s`", toChars());
 644                 const pkgfile = FileName.combine(FileName.removeExt(srcfile.toString()), package_d);
 645                 .errorSupplemental(loc, "Expected '%s' or '%s' in one of the following import paths:",
 646                     srcfile.toChars(), pkgfile.ptr);
 647             }
 648         }
 649         if (!global.gag)
 650         {
 651             /* Print path
 652              */
 653             if (global.path.length)
 654             {
 655                 foreach (i, p; global.path[])
 656                     fprintf(stderr, "import path[%llu] = %s\n", cast(ulong)i, p);
 657             }
 658             else
 659             {
 660                 fprintf(stderr, "Specify path to file '%s' with -I switch\n", srcfile.toChars());
 661             }
 662
 663             removeHdrFilesAndFail(global.params, Module.amodules);
 664         }
 665     }
 666
 667     /**
 668      * Reads the file from `srcfile` and loads the source buffer.
 669      *
 670      * If makefile module dependency is requested, we add this module
 671      * to the list of dependencies from here.
 672      *
 673      * Params:
 674      *  loc = the location
 675      *
 676      * Returns: `true` if successful
 677      */
 678     bool read(const ref Loc loc)
 679     {
 680         if (this.src)
 681             return true; // already read
 682
 683         //printf("Module::read('%s') file '%s'\n", toChars(), srcfile.toChars());
 684
 685         /* Preprocess the file if it's a .c file
 686          */
 687         FileName filename = srcfile;
 688
 689         const(ubyte)[] srctext;
 690         if (global.preprocess &&
 691             FileName.equalsExt(srcfile.toString(), c_ext) &&
 692             FileName.exists(srcfile.toString()))
 693         {
 694             /* Apply C preprocessor to the .c file, returning the contents
 695              * after preprocessing
 696              */
 697             srctext = global.preprocess(srcfile, loc, defines).data;
 698         }
 699         else
 700             srctext = global.fileManager.getFileContents(filename);
 701         this.src = srctext;
 702
 703         if (srctext)
 704         {
 705             if (global.params.makeDeps.doOutput)
 706                 global.params.makeDeps.files.push(srcfile.toChars());
 707             return true;
 708         }
 709
 710         this.onFileReadError(loc);
 711         return false;
 712     }
 713
 714     /// syntactic parse
 715     Module parse()
 716     {
 717         return parseModule!ASTCodegen();
 718     }
 719
 720     /// ditto
 721     extern (D) Module parseModule(AST)()
 722     {
 723         const(char)* srcname = srcfile.toChars();
 724         //printf("Module::parse(srcname = '%s')\n", srcname);
 725         isPackageFile = isPackageFileName(srcfile);
 726         const(char)[] buf = processSource(src, this);
 727         // an error happened on UTF conversion
 728         if (buf is null) return null;
 729
 730         /* If it starts with the string "Ddoc", then it's a documentation
 731          * source file.
 732          */
 733         if (buf.length>= 4 && buf[0..4] == "Ddoc")
 734         {
 735             comment = buf.ptr + 4;
 736             filetype = FileType.ddoc;
 737             if (!docfile)
 738                 setDocfile();
 739             return this;
 740         }
 741         /* If it has the extension ".dd", it is also a documentation
 742          * source file. Documentation source files may begin with "Ddoc"
 743          * but do not have to if they have the .dd extension.
 744          * https://issues.dlang.org/show_bug.cgi?id=15465
 745          */
 746         if (FileName.equalsExt(arg, dd_ext))
 747         {
 748             comment = buf.ptr; // the optional Ddoc, if present, is handled above.
 749             filetype = FileType.ddoc;
 750             if (!docfile)
 751                 setDocfile();
 752             return this;
 753         }
 754         /* If it has the extension ".di", it is a "header" file.
 755          */
 756         if (FileName.equalsExt(arg, hdr_ext))
 757             filetype = FileType.dhdr;
 758
 759         /// Promote `this` to a root module if requested via `-i`
 760         void checkCompiledImport()
 761         {
 762             if (!this.isRoot() && Compiler.onImport(this))
 763                 this.importedFrom = this;
 764         }
 765
 766         DsymbolTable dst;
 767         Package ppack = null;
 768
 769         /* If it has the extension ".c", it is a "C" file.
 770          * If it has the extension ".i", it is a preprocessed "C" file.
 771          */
 772         if (FileName.equalsExt(arg, c_ext) || FileName.equalsExt(arg, i_ext))
 773         {
 774             filetype = FileType.c;
 775
 776             global.compileEnv.masm = target.os == Target.OS.Windows && !target.omfobj; // Microsoft inline assembler format
 777             scope p = new CParser!AST(this, buf, cast(bool) docfile, global.errorSink, target.c, &defines, &global.compileEnv);
 778             global.compileEnv.masm = false;
 779             p.nextToken();
 780             checkCompiledImport();
 781             members = p.parseModule();
 782             assert(!p.md); // C doesn't have module declarations
 783             numlines = p.scanloc.linnum;
 784         }
 785         else
 786         {
 787             const bool doUnittests = global.params.parsingUnittestsRequired();
 788             scope p = new Parser!AST(this, buf, cast(bool) docfile, global.errorSink, &global.compileEnv, doUnittests);
 789             p.transitionIn = global.params.v.vin;
 790             p.nextToken();
 791             p.parseModuleDeclaration();
 792             md = p.md;
 793
 794             if (md)
 795             {
 796                 /* A ModuleDeclaration, md, was provided.
 797                 * The ModuleDeclaration sets the packages this module appears in, and
 798                 * the name of this module.
 799                 */
 800                 this.ident = md.id;
 801                 dst = Package.resolve(md.packages, &this.parent, &ppack);
 802             }
 803
 804             // Done after parsing the module header because `module x.y.z` may override the file name
 805             checkCompiledImport();
 806
 807             members = p.parseModuleContent();
 808             numlines = p.scanloc.linnum;
 809         }
 810
 811         /* The symbol table into which the module is to be inserted.
 812          */
 813
 814         if (md)
 815         {
 816             // Mark the package path as accessible from the current module
 817             // https://issues.dlang.org/show_bug.cgi?id=21661
 818             // Code taken from Import.addPackageAccess()
 819             if (md.packages.length > 0)
 820             {
 821                 // module a.b.c.d;
 822                 auto p = ppack; // a
 823                 addAccessiblePackage(p, Visibility(Visibility.Kind.private_));
 824                 foreach (id; md.packages[1 .. $]) // [b, c]
 825                 {
 826                     p = cast(Package) p.symtab.lookup(id);
 827                     if (p is null)
 828                         break;
 829                     addAccessiblePackage(p, Visibility(Visibility.Kind.private_));
 830                 }
 831             }
 832             assert(dst);
 833             Module m = ppack ? ppack.isModule() : null;
 834             if (m && !isPackageFileName(m.srcfile))
 835             {
 836                 .error(md.loc, "package name '%s' conflicts with usage as a module name in file %s", ppack.toPrettyChars(), m.srcfile.toChars());
 837             }
 838         }
 839         else
 840         {
 841             /* The name of the module is set to the source file name.
 842              * There are no packages.
 843              */
 844             dst = modules; // and so this module goes into global module symbol table
 845             /* Check to see if module name is a valid identifier
 846              */
 847             if (!Identifier.isValidIdentifier(this.ident.toChars()))
 848                 error(loc, "%s `%s` has non-identifier characters in filename, use module declaration instead", kind, toPrettyChars);
 849         }
 850         // Insert module into the symbol table
 851         Dsymbol s = this;
 852         if (isPackageFile)
 853         {
 854             /* If the source tree is as follows:
 855              *     pkg/
 856              *     +- package.d
 857              *     +- common.d
 858              * the 'pkg' will be incorporated to the internal package tree in two ways:
 859              *     import pkg;
 860              * and:
 861              *     import pkg.common;
 862              *
 863              * If both are used in one compilation, 'pkg' as a module (== pkg/package.d)
 864              * and a package name 'pkg' will conflict each other.
 865              *
 866              * To avoid the conflict:
 867              * 1. If preceding package name insertion had occurred by Package::resolve,
 868              *    reuse the previous wrapping 'Package' if it exists
 869              * 2. Otherwise, 'package.d' wrapped by 'Package' is inserted to the internal tree in here.
 870              *
 871              * Then change Package::isPkgMod to PKG.module_ and set Package::mod.
 872              *
 873              * Note that the 'wrapping Package' is the Package that contains package.d and other submodules,
 874              * the one inserted to the symbol table.
 875              */
 876             auto ps = dst.lookup(ident);
 877             Package p = ps ? ps.isPackage() : null;
 878             if (p is null)
 879             {
 880                 p = new Package(Loc.initial, ident);
 881                 p.tag = this.tag; // reuse the same package tag
 882                 p.symtab = new DsymbolTable();
 883             }
 884             this.tag = p.tag; // reuse the 'older' package tag
 885             this.pkg = p;
 886             p.parent = this.parent;
 887             p.isPkgMod = PKG.module_;
 888             p.mod = this;
 889             s = p;
 890         }
 891         if (!dst.insert(s))
 892         {
 893             /* It conflicts with a name that is already in the symbol table.
 894              * Figure out what went wrong, and issue error message.
 895              */
 896             Dsymbol prev = dst.lookup(ident);
 897             assert(prev);
 898             if (Module mprev = prev.isModule())
 899             {
 900                 if (!FileName.equals(srcname, mprev.srcfile.toChars()))
 901                     error(loc, "%s `%s` from file %s conflicts with another module %s from file %s", kind, toPrettyChars, srcname, mprev.toChars(), mprev.srcfile.toChars());
 902                 else if (isRoot() && mprev.isRoot())
 903                     error(loc, "%s `%s` from file %s is specified twice on the command line", kind, toPrettyChars, srcname);
 904                 else
 905                     error(loc, "%s `%s` from file %s must be imported with 'import %s;'", kind, toPrettyChars, srcname, toPrettyChars());
 906                 // https://issues.dlang.org/show_bug.cgi?id=14446
 907                 // Return previously parsed module to avoid AST duplication ICE.
 908                 return mprev;
 909             }
 910             else if (Package pkg = prev.isPackage())
 911             {
 912                 // 'package.d' loaded after a previous 'Package' insertion
 913                 if (isPackageFile)
 914                     amodules.push(this); // Add to global array of all modules
 915                 else
 916                     error(md ? md.loc : loc, "%s `%s` from file %s conflicts with package name %s", kind, toPrettyChars, srcname, pkg.toChars());
 917             }
 918             else
 919                 assert(global.errors);
 920         }
 921         else
 922         {
 923             // Add to global array of all modules
 924             amodules.push(this);
 925         }
 926         Compiler.onParseModule(this);
 927         return this;
 928     }
 929
 930     /**********************************
 931      * Determine if we need to generate an instance of ModuleInfo
 932      * for this Module.
 933      */
 934     int needModuleInfo()
 935     {
 936         //printf("needModuleInfo() %s, %d, %d\n", toChars(), needmoduleinfo, global.params.cov);
 937         return needmoduleinfo || global.params.cov;
 938     }
 939
 940     /*******************************************
 941      * Print deprecation warning if we're deprecated, when
 942      * this module is imported from scope sc.
 943      *
 944      * Params:
 945      *  sc = the scope into which we are imported
 946      *  loc = the location of the import statement
 947      */
 948     void checkImportDeprecation(const ref Loc loc, Scope* sc)
 949     {
 950         if (md && md.isdeprecated && !sc.isDeprecated)
 951         {
 952             Expression msg = md.msg;
 953             if (StringExp se = msg ? msg.toStringExp() : null)
 954             {
 955                 const slice = se.peekString();
 956                 if (slice.length)
 957                 {
 958                     deprecation(loc, "%s `%s` is deprecated - %.*s", kind, toPrettyChars, cast(int)slice.length, slice.ptr);
 959                     return;
 960                 }
 961             }
 962             deprecation(loc, "%s `%s` is deprecated", kind, toPrettyChars);
 963         }
 964     }
 965
 966     override bool isPackageAccessible(Package p, Visibility visibility, SearchOptFlags flags = SearchOpt.all)
 967     {
 968         if (insearch) // don't follow import cycles
 969             return false;
 970         insearch = true;
 971         scope (exit)
 972             insearch = false;
 973         if (flags & SearchOpt.ignorePrivateImports)
 974             visibility = Visibility(Visibility.Kind.public_); // only consider public imports
 975         return super.isPackageAccessible(p, visibility);
 976     }
 977
 978     override Dsymbol symtabInsert(Dsymbol s)
 979     {
 980         searchCacheIdent = null; // symbol is inserted, so invalidate cache
 981         return Package.symtabInsert(s);
 982     }
 983
 984     extern (D) void deleteObjFile()
 985     {
 986         if (global.params.obj)
 987             File.remove(objfile.toChars());
 988         if (docfile)
 989             File.remove(docfile.toChars());
 990     }
 991
 992     /*******************************************
 993      * Can't run semantic on s now, try again later.
 994      */
 995     extern (D) static void addDeferredSemantic(Dsymbol s)
 996     {
 997         //printf("Module::addDeferredSemantic('%s')\n", s.toChars());
 998         if (!deferred.contains(s))
 999             deferred.push(s);
1000     }
1001
1002     extern (D) static void addDeferredSemantic2(Dsymbol s)
1003     {
1004         //printf("Module::addDeferredSemantic2('%s')\n", s.toChars());
1005         if (!deferred2.contains(s))
1006             deferred2.push(s);
1007     }
1008
1009     extern (D) static void addDeferredSemantic3(Dsymbol s)
1010     {
1011         //printf("Module::addDeferredSemantic3('%s')\n", s.toChars());
1012         if (!deferred.contains(s))
1013             deferred3.push(s);
1014     }
1015
1016     /******************************************
1017      * Run semantic() on deferred symbols.
1018      */
1019     static void runDeferredSemantic()
1020     {
1021         __gshared int nested;
1022         if (nested)
1023             return;
1024         //if (deferred.length) printf("+Module::runDeferredSemantic(), len = %ld\n", deferred.length);
1025         nested++;
1026
1027         size_t len;
1028         do
1029         {
1030             len = deferred.length;
1031             if (!len)
1032                 break;
1033
1034             Dsymbol* todo;
1035             Dsymbol* todoalloc = null;
1036             Dsymbol tmp;
1037             if (len == 1)
1038             {
1039                 todo = &tmp;
1040             }
1041             else
1042             {
1043                 todo = cast(Dsymbol*)Mem.check(malloc(len * Dsymbol.sizeof));
1044                 todoalloc = todo;
1045             }
1046             memcpy(todo, deferred.tdata(), len * Dsymbol.sizeof);
1047             deferred.setDim(0);
1048
1049             foreach (i; 0..len)
1050             {
1051                 Dsymbol s = todo[i];
1052                 s.dsymbolSemantic(null);
1053                 //printf("deferred: %s, parent = %s\n", s.toChars(), s.parent.toChars());
1054             }
1055             //printf("\tdeferred.length = %ld, len = %ld\n", deferred.length, len);
1056             if (todoalloc)
1057                 free(todoalloc);
1058         }
1059         while (deferred.length != len); // while making progress
1060         nested--;
1061         //printf("-Module::runDeferredSemantic(), len = %ld\n", deferred.length);
1062     }
1063
1064     static void runDeferredSemantic2()
1065     {
1066         Module.runDeferredSemantic();
1067
1068         Dsymbols* a = &Module.deferred2;
1069         for (size_t i = 0; i < a.length; i++)
1070         {
1071             Dsymbol s = (*a)[i];
1072             //printf("[%d] %s semantic2a\n", i, s.toPrettyChars());
1073             s.semantic2(null);
1074
1075             if (global.errors)
1076                 break;
1077         }
1078         a.setDim(0);
1079     }
1080
1081     static void runDeferredSemantic3()
1082     {
1083         Module.runDeferredSemantic2();
1084
1085         Dsymbols* a = &Module.deferred3;
1086         for (size_t i = 0; i < a.length; i++)
1087         {
1088             Dsymbol s = (*a)[i];
1089             //printf("[%d] %s semantic3a\n", i, s.toPrettyChars());
1090             s.semantic3(null);
1091
1092             if (global.errors)
1093                 break;
1094         }
1095         a.setDim(0);
1096     }
1097
1098     extern (D) static void clearCache() nothrow
1099     {
1100         foreach (Module m; amodules)
1101             m.searchCacheIdent = null;
1102     }
1103
1104     /************************************
1105      * Recursively look at every module this module imports,
1106      * return true if it imports m.
1107      * Can be used to detect circular imports.
1108      */
1109     int imports(Module m) nothrow
1110     {
1111         //printf("%s Module::imports(%s)\n", toChars(), m.toChars());
1112         version (none)
1113         {
1114             foreach (i, Module mi; aimports)
1115                 printf("\t[%d] %s\n", cast(int) i, mi.toChars());
1116         }
1117         foreach (Module mi; aimports)
1118         {
1119             if (mi == m)
1120                 return true;
1121             if (!mi.insearch)
1122             {
1123                 mi.insearch = true;
1124                 int r = mi.imports(m);
1125                 if (r)
1126                     return r;
1127             }
1128         }
1129         return false;
1130     }
1131
1132     bool isRoot() nothrow
1133     {
1134         return this.importedFrom == this;
1135     }
1136
1137     /// Returns: Whether this module is in the `core` package and has name `ident`
1138     bool isCoreModule(Identifier ident) nothrow
1139     {
1140         return this.ident == ident && parent && parent.ident == Id.core && !parent.parent;
1141     }
1142
1143     // Back end
1144     int doppelganger; // sub-module
1145     Symbol* cov; // private uint[] __coverage;
1146     uint[] covb; // bit array of valid code line numbers
1147     Symbol* sictor; // module order independent constructor
1148     Symbol* sctor; // module constructor
1149     Symbol* sdtor; // module destructor
1150     Symbol* ssharedctor; // module shared constructor
1151     Symbol* sshareddtor; // module shared destructor
1152     Symbol* stest; // module unit test
1153     Symbol* sfilename; // symbol for filename
1154
1155     uint[uint] ctfe_cov; /// coverage information from ctfe execution_count[line]
1156
1157     override inout(Module) isModule() inout nothrow
1158     {
1159         return this;
1160     }
1161
1162     override void accept(Visitor v)
1163     {
1164         v.visit(this);
1165     }
1166
1167     /***********************************************
1168      * Writes this module's fully-qualified name to buf
1169      * Params:
1170      *    buf = The buffer to write to
1171      */
1172     void fullyQualifiedName(ref OutBuffer buf) nothrow
1173     {
1174         buf.writestring(ident.toString());
1175
1176         for (auto package_ = parent; package_ !is null; package_ = package_.parent)
1177         {
1178             buf.prependstring(".");
1179             buf.prependstring(package_.ident.toChars());
1180         }
1181     }
1182
1183     /** Lazily initializes and returns the escape table.
1184     Turns out it eats a lot of memory.
1185     */
1186     extern(D) Escape* escapetable() nothrow
1187     {
1188         if (!_escapetable)
1189             _escapetable = new Escape();
1190         return _escapetable;
1191     }
1192
1193     /****************************
1194      * A Singleton that loads core.stdc.config
1195      * Returns:
1196      *  Module of core.stdc.config, null if couldn't find it
1197      */
1198     extern (D) static Module loadCoreStdcConfig()
1199     {
1200         __gshared Module core_stdc_config;
1201         auto pkgids = new Identifier[2];
1202         pkgids[0] = Id.core;
1203         pkgids[1] = Id.stdc;
1204         return loadModuleFromLibrary(core_stdc_config, pkgids, Id.config);
1205     }
1206
1207     /****************************
1208      * A Singleton that loads core.atomic
1209      * Returns:
1210      *  Module of core.atomic, null if couldn't find it
1211      */
1212     extern (D) static Module loadCoreAtomic()
1213     {
1214         __gshared Module core_atomic;
1215         auto pkgids = new Identifier[1];
1216         pkgids[0] = Id.core;
1217         return loadModuleFromLibrary(core_atomic, pkgids, Id.atomic);
1218     }
1219
1220     /****************************
1221      * A Singleton that loads std.math
1222      * Returns:
1223      *  Module of std.math, null if couldn't find it
1224      */
1225     extern (D) static Module loadStdMath()
1226     {
1227         __gshared Module std_math;
1228         auto pkgids = new Identifier[1];
1229         pkgids[0] = Id.std;
1230         return loadModuleFromLibrary(std_math, pkgids, Id.math);
1231     }
1232
1233     /**********************************
1234      * Load a Module from the library.
1235      * Params:
1236      *  mod = cached return value of this call
1237      *  pkgids = package identifiers
1238      *  modid = module id
1239      * Returns:
1240      *  Module loaded, null if cannot load it
1241      */
1242     extern (D) private static Module loadModuleFromLibrary(ref Module mod, Identifier[] pkgids, Identifier modid)
1243     {
1244         if (mod)
1245             return mod;
1246
1247         auto imp = new Import(Loc.initial, pkgids[], modid, null, true);
1248         // Module.load will call fatal() if there's no module available.
1249         // Gag the error here, pushing the error handling to the caller.
1250         const errors = global.startGagging();
1251         imp.load(null);
1252         if (imp.mod)
1253         {
1254             imp.mod.importAll(null);
1255             imp.mod.dsymbolSemantic(null);
1256         }
1257         global.endGagging(errors);
1258         mod = imp.mod;
1259         return mod;
1260     }
1261 }
1262
1263 /***********************************************************
1264  */
1265 extern (C++) struct ModuleDeclaration
1266 {
1267     Loc loc;
1268     Identifier id;
1269     Identifier[] packages;  // array of Identifier's representing packages
1270     bool isdeprecated;      // if it is a deprecated module
1271     Expression msg;
1272
1273     extern (D) this(const ref Loc loc, Identifier[] packages, Identifier id, Expression msg, bool isdeprecated) @safe
1274     {
1275         this.loc = loc;
1276         this.packages = packages;
1277         this.id = id;
1278         this.msg = msg;
1279         this.isdeprecated = isdeprecated;
1280     }
1281
1282     extern (C++) const(char)* toChars() const @safe
1283     {
1284         OutBuffer buf;
1285         foreach (pid; packages)
1286         {
1287             buf.writestring(pid.toString());
1288             buf.writeByte('.');
1289         }
1290         buf.writestring(id.toString());
1291         return buf.extractChars();
1292     }
1293
1294     /// Provide a human readable representation
1295     extern (D) const(char)[] toString() const
1296     {
1297         return this.toChars().toDString;
1298     }
1299 }
1300
1301 /****************************************
1302  * Create array of the local classes in the Module, suitable
1303  * for inclusion in ModuleInfo
1304  * Params:
1305  *      mod = the Module
1306  *      aclasses = array to fill in
1307  * Returns: array of local classes
1308  */
1309 void getLocalClasses(Module mod, ref ClassDeclarations aclasses)
1310 {
1311     //printf("members.length = %d\n", mod.members.length);
1312     int pushAddClassDg(size_t n, Dsymbol sm)
1313     {
1314         if (!sm)
1315             return 0;
1316
1317         if (auto cd = sm.isClassDeclaration())
1318         {
1319             // compatibility with previous algorithm
1320             if (cd.parent && cd.parent.isTemplateMixin())
1321                 return 0;
1322
1323             if (cd.classKind != ClassKind.objc)
1324                 aclasses.push(cd);
1325         }
1326         return 0;
1327     }
1328
1329     _foreach(null, mod.members, &pushAddClassDg);
1330 }
1331
1332
1333 alias ForeachDg = int delegate(size_t idx, Dsymbol s);
1334
1335 /***************************************
1336  * Expands attribute declarations in members in depth first
1337  * order. Calls dg(size_t symidx, Dsymbol *sym) for each
1338  * member.
1339  * If dg returns !=0, stops and returns that value else returns 0.
1340  * Use this function to avoid the O(N + N^2/2) complexity of
1341  * calculating dim and calling N times getNth.
1342  * Returns:
1343  *  last value returned by dg()
1344  */
1345 int _foreach(Scope* sc, Dsymbols* members, scope ForeachDg dg, size_t* pn = null)
1346 {
1347     assert(dg);
1348     if (!members)
1349         return 0;
1350     size_t n = pn ? *pn : 0; // take over index
1351     int result = 0;
1352     foreach (size_t i; 0 .. members.length)
1353     {
1354         import dmd.attrib : AttribDeclaration;
1355         import dmd.dtemplate : TemplateMixin;
1356
1357         Dsymbol s = (*members)[i];
1358         if (AttribDeclaration a = s.isAttribDeclaration())
1359             result = _foreach(sc, a.include(sc), dg, &n);
1360         else if (TemplateMixin tm = s.isTemplateMixin())
1361             result = _foreach(sc, tm.members, dg, &n);
1362         else if (s.isTemplateInstance())
1363         {
1364         }
1365         else if (s.isUnitTestDeclaration())
1366         {
1367         }
1368         else
1369             result = dg(n++, s);
1370         if (result)
1371             break;
1372     }
1373     if (pn)
1374         *pn = n; // update index
1375     return result;
1376 }
1377
1378 /**
1379  * Process the content of a source file
1380  *
1381  * Attempts to find which encoding it is using, if it has BOM,
1382  * and then normalize the source to UTF-8. If no encoding is required,
1383  * a slice of `src` will be returned without extra allocation.
1384  *
1385  * Params:
1386  *  src = Content of the source file to process
1387  *  mod = Module matching `src`, used for error handling
1388  *
1389  * Returns:
1390  *   UTF-8 encoded variant of `src`, stripped of any BOM,
1391  *   or `null` if an error happened.
1392  */
1393 private const(char)[] processSource (const(ubyte)[] src, Module mod)
1394 {
1395     enum SourceEncoding { utf16, utf32}
1396     enum Endian { little, big}
1397
1398     /*
1399      * Convert a buffer from UTF32 to UTF8
1400      * Params:
1401      *    Endian = is the buffer big/little endian
1402      *    buf = buffer of UTF32 data
1403      * Returns:
1404      *    input buffer reencoded as UTF8
1405      */
1406
1407     char[] UTF32ToUTF8(Endian endian)(const(char)[] buf)
1408     {
1409         static if (endian == Endian.little)
1410             alias readNext = Port.readlongLE;
1411         else
1412             alias readNext = Port.readlongBE;
1413
1414         if (buf.length & 3)
1415         {
1416             .error(mod.loc, "%s `%s` odd length of UTF-32 char source %llu",
1417                 mod.kind, mod.toPrettyChars, cast(ulong) buf.length);
1418             return null;
1419         }
1420
1421         const (uint)[] eBuf = cast(const(uint)[])buf;
1422
1423         OutBuffer dbuf;
1424         dbuf.reserve(eBuf.length);
1425
1426         foreach (i; 0 .. eBuf.length)
1427         {
1428             const u = readNext(&eBuf[i]);
1429             if (u & ~0x7F)
1430             {
1431                 if (u > 0x10FFFF)
1432                 {
1433                     .error(mod.loc, "%s `%s` UTF-32 value %08x greater than 0x10FFFF", mod.kind, mod.toPrettyChars, u);
1434                     return null;
1435                 }
1436                 dbuf.writeUTF8(u);
1437             }
1438             else
1439                 dbuf.writeByte(u);
1440         }
1441         dbuf.writeByte(0); //add null terminator
1442         return dbuf.extractSlice();
1443     }
1444
1445     /*
1446      * Convert a buffer from UTF16 to UTF8
1447      * Params:
1448      *    Endian = is the buffer big/little endian
1449      *    buf = buffer of UTF16 data
1450      * Returns:
1451      *    input buffer reencoded as UTF8
1452      */
1453
1454     char[] UTF16ToUTF8(Endian endian)(const(char)[] buf)
1455     {
1456         static if (endian == Endian.little)
1457             alias readNext = Port.readwordLE;
1458         else
1459             alias readNext = Port.readwordBE;
1460
1461         if (buf.length & 1)
1462         {
1463             .error(mod.loc, "%s `%s` odd length of UTF-16 char source %llu", mod.kind, mod.toPrettyChars, cast(ulong) buf.length);
1464             return null;
1465         }
1466
1467         const (ushort)[] eBuf = cast(const(ushort)[])buf;
1468
1469         OutBuffer dbuf;
1470         dbuf.reserve(eBuf.length);
1471
1472         //i will be incremented in the loop for high codepoints
1473         foreach (ref i; 0 .. eBuf.length)
1474         {
1475             uint u = readNext(&eBuf[i]);
1476             if (u & ~0x7F)
1477             {
1478                 if (0xD800 <= u && u < 0xDC00)
1479                 {
1480                     i++;
1481                     if (i >= eBuf.length)
1482                     {
1483                         .error(mod.loc, "%s `%s` surrogate UTF-16 high value %04x at end of file", mod.kind, mod.toPrettyChars, u);
1484                         return null;
1485                     }
1486                     const u2 = readNext(&eBuf[i]);
1487                     if (u2 < 0xDC00 || 0xE000 <= u2)
1488                     {
1489                         .error(mod.loc, "%s `%s` surrogate UTF-16 low value %04x out of range", mod.kind, mod.toPrettyChars, u2);
1490                         return null;
1491                     }
1492                     u = (u - 0xD7C0) << 10;
1493                     u |= (u2 - 0xDC00);
1494                 }
1495                 else if (u >= 0xDC00 && u <= 0xDFFF)
1496                 {
1497                     .error(mod.loc, "%s `%s` unpaired surrogate UTF-16 value %04x", mod.kind, mod.toPrettyChars, u);
1498                     return null;
1499                 }
1500                 else if (u == 0xFFFE || u == 0xFFFF)
1501                 {
1502                     .error(mod.loc, "%s `%s` illegal UTF-16 value %04x", mod.kind, mod.toPrettyChars, u);
1503                     return null;
1504                 }
1505                 dbuf.writeUTF8(u);
1506             }
1507             else
1508                 dbuf.writeByte(u);
1509         }
1510         dbuf.writeByte(0); //add a terminating null byte
1511         return dbuf.extractSlice();
1512     }
1513
1514     const(char)[] buf = cast(const(char)[]) src;
1515
1516     // Assume the buffer is from memory and has not be read from disk. Assume UTF-8.
1517     if (buf.length < 2)
1518         return buf;
1519
1520     /* Convert all non-UTF-8 formats to UTF-8.
1521      * BOM : https://www.unicode.org/faq/utf_bom.html
1522      * 00 00 FE FF  UTF-32BE, big-endian
1523      * FF FE 00 00  UTF-32LE, little-endian
1524      * FE FF        UTF-16BE, big-endian
1525      * FF FE        UTF-16LE, little-endian
1526      * EF BB BF     UTF-8
1527      */
1528     if (buf[0] == 0xFF && buf[1] == 0xFE)
1529     {
1530         if (buf.length >= 4 && buf[2] == 0 && buf[3] == 0)
1531             return UTF32ToUTF8!(Endian.little)(buf[4 .. $]);
1532         return UTF16ToUTF8!(Endian.little)(buf[2 .. $]);
1533     }
1534
1535     if (buf[0] == 0xFE && buf[1] == 0xFF)
1536         return UTF16ToUTF8!(Endian.big)(buf[2 .. $]);
1537
1538     if (buf.length >= 4 && buf[0] == 0 && buf[1] == 0 && buf[2] == 0xFE && buf[3] == 0xFF)
1539         return UTF32ToUTF8!(Endian.big)(buf[4 .. $]);
1540
1541     if (buf.length >= 3 && buf[0] == 0xEF && buf[1] == 0xBB && buf[2] == 0xBF)
1542         return buf[3 .. $];
1543
1544     /* There is no BOM. Make use of Arcane Jill's insight that
1545      * the first char of D source must be ASCII to
1546      * figure out the encoding.
1547      */
1548     if (buf.length >= 4 && buf[1] == 0 && buf[2] == 0 && buf[3] == 0)
1549         return UTF32ToUTF8!(Endian.little)(buf);
1550     if (buf.length >= 4 && buf[0] == 0 && buf[1] == 0 && buf[2] == 0)
1551         return UTF32ToUTF8!(Endian.big)(buf);
1552     // try to check for UTF-16
1553     if (buf.length >= 2 && buf[1] == 0)
1554         return UTF16ToUTF8!(Endian.little)(buf);
1555     if (buf[0] == 0)
1556         return UTF16ToUTF8!(Endian.big)(buf);
1557
1558     // It's UTF-8
1559     if (buf[0] >= 0x80)
1560     {
1561         auto loc = mod.getLoc();
1562         .error(loc, "%s `%s` source file must start with BOM or ASCII character, not \\x%02X", mod.kind, mod.toPrettyChars, buf[0]);
1563         return null;
1564     }
1565
1566     return buf;
1567 }
1568
1569 /*******************************************
1570  * Look for member of the form:
1571  *      const(MemberInfo)[] getMembers(string);
1572  * Returns NULL if not found
1573  */
1574 FuncDeclaration findGetMembers(ScopeDsymbol dsym)
1575 {
1576     import dmd.opover : search_function;
1577     Dsymbol s = search_function(dsym, Id.getmembers);
1578     FuncDeclaration fdx = s ? s.isFuncDeclaration() : null;
1579     version (none)
1580     {
1581         // Finish
1582         __gshared TypeFunction tfgetmembers;
1583         if (!tfgetmembers)
1584         {
1585             Scope sc;
1586             sc.eSink = global.errorSink;
1587             auto parameters = new Parameters();
1588             Parameters* p = new Parameter(STC.in_, Type.tchar.constOf().arrayOf(), null, null);
1589             parameters.push(p);
1590             Type tret = null;
1591             TypeFunction tf = new TypeFunction(parameters, tret, VarArg.none, LINK.d);
1592             tfgetmembers = tf.dsymbolSemantic(Loc.initial, &sc).isTypeFunction();
1593         }
1594         if (fdx)
1595             fdx = fdx.overloadExactMatch(tfgetmembers);
1596     }
1597     if (fdx && fdx.isVirtual())
1598         fdx = null;
1599     return fdx;
1600 }