head 1.1; branch 1.1.1; access; symbols netbsd-11-0-RC5:1.1.1.9 netbsd-11-0-RC4:1.1.1.9 netbsd-11-0-RC3:1.1.1.9 netbsd-11-0-RC2:1.1.1.9 netbsd-11-0-RC1:1.1.1.9 perseant-exfatfs-base-20250801:1.1.1.9 netbsd-11:1.1.1.9.0.10 netbsd-11-base:1.1.1.9 netbsd-10-1-RELEASE:1.1.1.9 perseant-exfatfs-base-20240630:1.1.1.9 perseant-exfatfs:1.1.1.9.0.8 perseant-exfatfs-base:1.1.1.9 netbsd-8-3-RELEASE:1.1.1.7 netbsd-9-4-RELEASE:1.1.1.8 netbsd-10-0-RELEASE:1.1.1.9 netbsd-10-0-RC6:1.1.1.9 netbsd-10-0-RC5:1.1.1.9 netbsd-10-0-RC4:1.1.1.9 netbsd-10-0-RC3:1.1.1.9 netbsd-10-0-RC2:1.1.1.9 netbsd-10-0-RC1:1.1.1.9 netbsd-10:1.1.1.9.0.6 netbsd-10-base:1.1.1.9 netbsd-9-3-RELEASE:1.1.1.8 cjep_sun2x:1.1.1.9.0.4 cjep_sun2x-base:1.1.1.9 cjep_staticlib_x-base1:1.1.1.9 netbsd-9-2-RELEASE:1.1.1.8 cjep_staticlib_x:1.1.1.9.0.2 cjep_staticlib_x-base:1.1.1.9 netbsd-9-1-RELEASE:1.1.1.8 phil-wifi-20200421:1.1.1.9 phil-wifi-20200411:1.1.1.9 phil-wifi-20200406:1.1.1.9 netbsd-8-2-RELEASE:1.1.1.7 netbsd-9-0-RELEASE:1.1.1.8 netbsd-9-0-RC2:1.1.1.8 netbsd-9-0-RC1:1.1.1.8 netbsd-9:1.1.1.8.0.2 netbsd-9-base:1.1.1.8 phil-wifi-20190609:1.1.1.8 netbsd-8-1-RELEASE:1.1.1.7 netbsd-8-1-RC1:1.1.1.7 pgoyette-compat-merge-20190127:1.1.1.7.12.1 pgoyette-compat-20190127:1.1.1.8 pgoyette-compat-20190118:1.1.1.8 pgoyette-compat-1226:1.1.1.8 pgoyette-compat-1126:1.1.1.8 pgoyette-compat-1020:1.1.1.8 pgoyette-compat-0930:1.1.1.8 pgoyette-compat-0906:1.1.1.8 netbsd-7-2-RELEASE:1.1.1.4.2.1 pgoyette-compat-0728:1.1.1.8 clang-337282:1.1.1.8 netbsd-8-0-RELEASE:1.1.1.7 phil-wifi:1.1.1.7.0.14 phil-wifi-base:1.1.1.7 pgoyette-compat-0625:1.1.1.7 netbsd-8-0-RC2:1.1.1.7 pgoyette-compat-0521:1.1.1.7 pgoyette-compat-0502:1.1.1.7 pgoyette-compat-0422:1.1.1.7 netbsd-8-0-RC1:1.1.1.7 pgoyette-compat-0415:1.1.1.7 pgoyette-compat-0407:1.1.1.7 pgoyette-compat-0330:1.1.1.7 pgoyette-compat-0322:1.1.1.7 pgoyette-compat-0315:1.1.1.7 netbsd-7-1-2-RELEASE:1.1.1.4.2.1 pgoyette-compat:1.1.1.7.0.12 pgoyette-compat-base:1.1.1.7 netbsd-7-1-1-RELEASE:1.1.1.4.2.1 clang-319952:1.1.1.7 matt-nb8-mediatek:1.1.1.7.0.10 matt-nb8-mediatek-base:1.1.1.7 clang-309604:1.1.1.7 perseant-stdc-iso10646:1.1.1.7.0.8 perseant-stdc-iso10646-base:1.1.1.7 netbsd-8:1.1.1.7.0.6 netbsd-8-base:1.1.1.7 prg-localcount2-base3:1.1.1.7 prg-localcount2-base2:1.1.1.7 prg-localcount2-base1:1.1.1.7 prg-localcount2:1.1.1.7.0.4 prg-localcount2-base:1.1.1.7 pgoyette-localcount-20170426:1.1.1.7 bouyer-socketcan-base1:1.1.1.7 pgoyette-localcount-20170320:1.1.1.7 netbsd-7-1:1.1.1.4.2.1.0.6 netbsd-7-1-RELEASE:1.1.1.4.2.1 netbsd-7-1-RC2:1.1.1.4.2.1 clang-294123:1.1.1.7 netbsd-7-nhusb-base-20170116:1.1.1.4.2.1 bouyer-socketcan:1.1.1.7.0.2 bouyer-socketcan-base:1.1.1.7 clang-291444:1.1.1.7 pgoyette-localcount-20170107:1.1.1.6 netbsd-7-1-RC1:1.1.1.4.2.1 pgoyette-localcount-20161104:1.1.1.6 netbsd-7-0-2-RELEASE:1.1.1.4.2.1 localcount-20160914:1.1.1.6 netbsd-7-nhusb:1.1.1.4.2.1.0.4 netbsd-7-nhusb-base:1.1.1.4.2.1 clang-280599:1.1.1.6 pgoyette-localcount-20160806:1.1.1.6 pgoyette-localcount-20160726:1.1.1.6 pgoyette-localcount:1.1.1.6.0.2 pgoyette-localcount-base:1.1.1.6 netbsd-7-0-1-RELEASE:1.1.1.4.2.1 clang-261930:1.1.1.6 netbsd-7-0:1.1.1.4.2.1.0.2 netbsd-7-0-RELEASE:1.1.1.4.2.1 netbsd-7-0-RC3:1.1.1.4.2.1 netbsd-7-0-RC2:1.1.1.4.2.1 netbsd-7-0-RC1:1.1.1.4.2.1 clang-237755:1.1.1.5 clang-232565:1.1.1.5 clang-227398:1.1.1.5 tls-maxphys-base:1.1.1.4 tls-maxphys:1.1.1.4.0.4 netbsd-7:1.1.1.4.0.2 netbsd-7-base:1.1.1.4 clang-215315:1.1.1.4 clang-209886:1.1.1.3 yamt-pagecache:1.1.1.2.0.4 yamt-pagecache-base9:1.1.1.2 tls-earlyentropy:1.1.1.2.0.2 tls-earlyentropy-base:1.1.1.3 riastradh-xf86-video-intel-2-7-1-pre-2-21-15:1.1.1.2 riastradh-drm2-base3:1.1.1.2 clang-202566:1.1.1.2 clang-201163:1.1.1.1 clang-199312:1.1.1.1 clang-198450:1.1.1.1 clang-196603:1.1.1.1 clang-195771:1.1.1.1 LLVM:1.1.1; locks; strict; comment @// @; 1.1 date 2013.11.28.14.14.49; author joerg; state Exp; branches 1.1.1.1; next ; commitid ow8OybrawrB1f3fx; 1.1.1.1 date 2013.11.28.14.14.49; author joerg; state Exp; branches; next 1.1.1.2; commitid ow8OybrawrB1f3fx; 1.1.1.2 date 2014.03.04.19.53.39; author joerg; state Exp; branches 1.1.1.2.2.1 1.1.1.2.4.1; next 1.1.1.3; commitid 29z1hJonZISIXprx; 1.1.1.3 date 2014.05.30.18.14.41; author joerg; state Exp; branches; next 1.1.1.4; commitid 8q0kdlBlCn09GACx; 1.1.1.4 date 2014.08.10.17.08.26; author joerg; state Exp; branches 1.1.1.4.2.1 1.1.1.4.4.1; next 1.1.1.5; commitid N85tXAN6Ex9VZPLx; 1.1.1.5 date 2015.01.29.19.57.32; author joerg; state Exp; branches; next 1.1.1.6; commitid mlISSizlPKvepX7y; 1.1.1.6 date 2016.02.27.22.11.52; author joerg; state Exp; branches 1.1.1.6.2.1; next 1.1.1.7; commitid tIimz3oDlh1NpBWy; 1.1.1.7 date 2017.01.11.10.34.48; author joerg; state Exp; branches 1.1.1.7.12.1 1.1.1.7.14.1; next 1.1.1.8; commitid CNnUNfII1jgNmxBz; 1.1.1.8 date 2018.07.17.18.31.09; author joerg; state Exp; branches; next 1.1.1.9; commitid wDzL46ALjrCZgwKA; 1.1.1.9 date 2019.11.13.22.19.23; author joerg; state dead; branches; next ; commitid QD8YATxuNG34YJKB; 1.1.1.2.2.1 date 2014.08.10.07.08.08; author tls; state Exp; branches; next ; commitid t01A1TLTYxkpGMLx; 1.1.1.2.4.1 date 2014.03.04.19.53.39; author yamt; state dead; branches; next 1.1.1.2.4.2; commitid WSrDtL5nYAUyiyBx; 1.1.1.2.4.2 date 2014.05.22.16.18.27; author yamt; state Exp; branches; next ; commitid WSrDtL5nYAUyiyBx; 1.1.1.4.2.1 date 2015.06.04.20.04.28; author snj; state Exp; branches; next ; commitid yRnjq9fueSo6n9oy; 1.1.1.4.4.1 date 2014.08.10.17.08.26; author tls; state dead; branches; next 1.1.1.4.4.2; commitid jTnpym9Qu0o4R1Nx; 1.1.1.4.4.2 date 2014.08.19.23.47.28; author tls; state Exp; branches; next ; commitid jTnpym9Qu0o4R1Nx; 1.1.1.6.2.1 date 2017.03.20.06.52.38; author pgoyette; state Exp; branches; next ; commitid jjw7cAwgyKq7RfKz; 1.1.1.7.12.1 date 2018.07.28.04.33.19; author pgoyette; state Exp; branches; next ; commitid 1UP1xAIUxv1ZgRLA; 1.1.1.7.14.1 date 2019.06.10.21.45.23; author christos; state Exp; branches; next 1.1.1.7.14.2; commitid jtc8rnCzWiEEHGqB; 1.1.1.7.14.2 date 2020.04.13.07.46.34; author martin; state dead; branches; next ; commitid X01YhRUPVUDaec4C; desc @@ 1.1 log @Initial revision @ text @//===--- CacheTokens.cpp - Caching of lexer tokens for PTH support --------===// // // The LLVM Compiler Infrastructure // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// // // This provides a possible implementation of PTH support for Clang that is // based on caching lexed tokens and identifiers. // //===----------------------------------------------------------------------===// #include "clang/Frontend/Utils.h" #include "clang/Basic/Diagnostic.h" #include "clang/Basic/FileManager.h" #include "clang/Basic/FileSystemStatCache.h" #include "clang/Basic/IdentifierTable.h" #include "clang/Basic/OnDiskHashTable.h" #include "clang/Basic/SourceManager.h" #include "clang/Lex/Lexer.h" #include "clang/Lex/Preprocessor.h" #include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringMap.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Path.h" #include "llvm/Support/raw_ostream.h" // FIXME: put this somewhere else? #ifndef S_ISDIR #define S_ISDIR(x) (((x)&_S_IFDIR)!=0) #endif using namespace clang; using namespace clang::io; //===----------------------------------------------------------------------===// // PTH-specific stuff. //===----------------------------------------------------------------------===// namespace { class PTHEntry { Offset TokenData, PPCondData; public: PTHEntry() {} PTHEntry(Offset td, Offset ppcd) : TokenData(td), PPCondData(ppcd) {} Offset getTokenOffset() const { return TokenData; } Offset getPPCondTableOffset() const { return PPCondData; } }; class PTHEntryKeyVariant { union { const FileEntry* FE; const char* Path; }; enum { IsFE = 0x1, IsDE = 0x2, IsNoExist = 0x0 } Kind; FileData *Data; public: PTHEntryKeyVariant(const FileEntry *fe) : FE(fe), Kind(IsFE), Data(0) {} PTHEntryKeyVariant(FileData *Data, const char *path) : Path(path), Kind(IsDE), Data(new FileData(*Data)) {} explicit PTHEntryKeyVariant(const char *path) : Path(path), Kind(IsNoExist), Data(0) {} bool isFile() const { return Kind == IsFE; } StringRef getString() const { return Kind == IsFE ? FE->getName() : Path; } unsigned getKind() const { return (unsigned) Kind; } void EmitData(raw_ostream& Out) { switch (Kind) { case IsFE: { // Emit stat information. llvm::sys::fs::UniqueID UID = FE->getUniqueID(); ::Emit64(Out, UID.getFile()); ::Emit64(Out, UID.getDevice()); ::Emit64(Out, FE->getModificationTime()); ::Emit64(Out, FE->getSize()); } break; case IsDE: // Emit stat information. ::Emit64(Out, Data->UniqueID.getFile()); ::Emit64(Out, Data->UniqueID.getDevice()); ::Emit64(Out, Data->ModTime); ::Emit64(Out, Data->Size); delete Data; break; default: break; } } unsigned getRepresentationLength() const { return Kind == IsNoExist ? 0 : 4 + 4 + 2 + 8 + 8; } }; class FileEntryPTHEntryInfo { public: typedef PTHEntryKeyVariant key_type; typedef key_type key_type_ref; typedef PTHEntry data_type; typedef const PTHEntry& data_type_ref; static unsigned ComputeHash(PTHEntryKeyVariant V) { return llvm::HashString(V.getString()); } static std::pair EmitKeyDataLength(raw_ostream& Out, PTHEntryKeyVariant V, const PTHEntry& E) { unsigned n = V.getString().size() + 1 + 1; ::Emit16(Out, n); unsigned m = V.getRepresentationLength() + (V.isFile() ? 4 + 4 : 0); ::Emit8(Out, m); return std::make_pair(n, m); } static void EmitKey(raw_ostream& Out, PTHEntryKeyVariant V, unsigned n){ // Emit the entry kind. ::Emit8(Out, (unsigned) V.getKind()); // Emit the string. Out.write(V.getString().data(), n - 1); } static void EmitData(raw_ostream& Out, PTHEntryKeyVariant V, const PTHEntry& E, unsigned) { // For file entries emit the offsets into the PTH file for token data // and the preprocessor blocks table. if (V.isFile()) { ::Emit32(Out, E.getTokenOffset()); ::Emit32(Out, E.getPPCondTableOffset()); } // Emit any other data associated with the key (i.e., stat information). V.EmitData(Out); } }; class OffsetOpt { bool valid; Offset off; public: OffsetOpt() : valid(false) {} bool hasOffset() const { return valid; } Offset getOffset() const { assert(valid); return off; } void setOffset(Offset o) { off = o; valid = true; } }; } // end anonymous namespace typedef OnDiskChainedHashTableGenerator PTHMap; namespace { class PTHWriter { typedef llvm::DenseMap IDMap; typedef llvm::StringMap CachedStrsTy; IDMap IM; llvm::raw_fd_ostream& Out; Preprocessor& PP; uint32_t idcount; PTHMap PM; CachedStrsTy CachedStrs; Offset CurStrOffset; std::vector*> StrEntries; //// Get the persistent id for the given IdentifierInfo*. uint32_t ResolveID(const IdentifierInfo* II); /// Emit a token to the PTH file. void EmitToken(const Token& T); void Emit8(uint32_t V) { ::Emit8(Out, V); } void Emit16(uint32_t V) { ::Emit16(Out, V); } void Emit32(uint32_t V) { ::Emit32(Out, V); } void EmitBuf(const char *Ptr, unsigned NumBytes) { Out.write(Ptr, NumBytes); } void EmitString(StringRef V) { ::Emit16(Out, V.size()); EmitBuf(V.data(), V.size()); } /// EmitIdentifierTable - Emits two tables to the PTH file. The first is /// a hashtable mapping from identifier strings to persistent IDs. /// The second is a straight table mapping from persistent IDs to string data /// (the keys of the first table). std::pair EmitIdentifierTable(); /// EmitFileTable - Emit a table mapping from file name strings to PTH /// token data. Offset EmitFileTable() { return PM.Emit(Out); } PTHEntry LexTokens(Lexer& L); Offset EmitCachedSpellings(); public: PTHWriter(llvm::raw_fd_ostream& out, Preprocessor& pp) : Out(out), PP(pp), idcount(0), CurStrOffset(0) {} PTHMap &getPM() { return PM; } void GeneratePTH(const std::string &MainFile); }; } // end anonymous namespace uint32_t PTHWriter::ResolveID(const IdentifierInfo* II) { // Null IdentifierInfo's map to the persistent ID 0. if (!II) return 0; IDMap::iterator I = IM.find(II); if (I != IM.end()) return I->second; // We've already added 1. IM[II] = ++idcount; // Pre-increment since '0' is reserved for NULL. return idcount; } void PTHWriter::EmitToken(const Token& T) { // Emit the token kind, flags, and length. Emit32(((uint32_t) T.getKind()) | ((((uint32_t) T.getFlags())) << 8)| (((uint32_t) T.getLength()) << 16)); if (!T.isLiteral()) { Emit32(ResolveID(T.getIdentifierInfo())); } else { // We cache *un-cleaned* spellings. This gives us 100% fidelity with the // source code. StringRef s(T.getLiteralData(), T.getLength()); // Get the string entry. llvm::StringMapEntry *E = &CachedStrs.GetOrCreateValue(s); // If this is a new string entry, bump the PTH offset. if (!E->getValue().hasOffset()) { E->getValue().setOffset(CurStrOffset); StrEntries.push_back(E); CurStrOffset += s.size() + 1; } // Emit the relative offset into the PTH file for the spelling string. Emit32(E->getValue().getOffset()); } // Emit the offset into the original source file of this token so that we // can reconstruct its SourceLocation. Emit32(PP.getSourceManager().getFileOffset(T.getLocation())); } PTHEntry PTHWriter::LexTokens(Lexer& L) { // Pad 0's so that we emit tokens to a 4-byte alignment. // This speed up reading them back in. Pad(Out, 4); Offset TokenOff = (Offset) Out.tell(); // Keep track of matching '#if' ... '#endif'. typedef std::vector > PPCondTable; PPCondTable PPCond; std::vector PPStartCond; bool ParsingPreprocessorDirective = false; Token Tok; do { L.LexFromRawLexer(Tok); NextToken: if ((Tok.isAtStartOfLine() || Tok.is(tok::eof)) && ParsingPreprocessorDirective) { // Insert an eod token into the token cache. It has the same // position as the next token that is not on the same line as the // preprocessor directive. Observe that we continue processing // 'Tok' when we exit this branch. Token Tmp = Tok; Tmp.setKind(tok::eod); Tmp.clearFlag(Token::StartOfLine); Tmp.setIdentifierInfo(0); EmitToken(Tmp); ParsingPreprocessorDirective = false; } if (Tok.is(tok::raw_identifier)) { PP.LookUpIdentifierInfo(Tok); EmitToken(Tok); continue; } if (Tok.is(tok::hash) && Tok.isAtStartOfLine()) { // Special processing for #include. Store the '#' token and lex // the next token. assert(!ParsingPreprocessorDirective); Offset HashOff = (Offset) Out.tell(); // Get the next token. Token NextTok; L.LexFromRawLexer(NextTok); // If we see the start of line, then we had a null directive "#". In // this case, discard both tokens. if (NextTok.isAtStartOfLine()) goto NextToken; // The token is the start of a directive. Emit it. EmitToken(Tok); Tok = NextTok; // Did we see 'include'/'import'/'include_next'? if (Tok.isNot(tok::raw_identifier)) { EmitToken(Tok); continue; } IdentifierInfo* II = PP.LookUpIdentifierInfo(Tok); tok::PPKeywordKind K = II->getPPKeywordID(); ParsingPreprocessorDirective = true; switch (K) { case tok::pp_not_keyword: // Invalid directives "#foo" can occur in #if 0 blocks etc, just pass // them through. default: break; case tok::pp_include: case tok::pp_import: case tok::pp_include_next: { // Save the 'include' token. EmitToken(Tok); // Lex the next token as an include string. L.setParsingPreprocessorDirective(true); L.LexIncludeFilename(Tok); L.setParsingPreprocessorDirective(false); assert(!Tok.isAtStartOfLine()); if (Tok.is(tok::raw_identifier)) PP.LookUpIdentifierInfo(Tok); break; } case tok::pp_if: case tok::pp_ifdef: case tok::pp_ifndef: { // Add an entry for '#if' and friends. We initially set the target // index to 0. This will get backpatched when we hit #endif. PPStartCond.push_back(PPCond.size()); PPCond.push_back(std::make_pair(HashOff, 0U)); break; } case tok::pp_endif: { // Add an entry for '#endif'. We set the target table index to itself. // This will later be set to zero when emitting to the PTH file. We // use 0 for uninitialized indices because that is easier to debug. unsigned index = PPCond.size(); // Backpatch the opening '#if' entry. assert(!PPStartCond.empty()); assert(PPCond.size() > PPStartCond.back()); assert(PPCond[PPStartCond.back()].second == 0); PPCond[PPStartCond.back()].second = index; PPStartCond.pop_back(); // Add the new entry to PPCond. PPCond.push_back(std::make_pair(HashOff, index)); EmitToken(Tok); // Some files have gibberish on the same line as '#endif'. // Discard these tokens. do L.LexFromRawLexer(Tok); while (Tok.isNot(tok::eof) && !Tok.isAtStartOfLine()); // We have the next token in hand. // Don't immediately lex the next one. goto NextToken; } case tok::pp_elif: case tok::pp_else: { // Add an entry for #elif or #else. // This serves as both a closing and opening of a conditional block. // This means that its entry will get backpatched later. unsigned index = PPCond.size(); // Backpatch the previous '#if' entry. assert(!PPStartCond.empty()); assert(PPCond.size() > PPStartCond.back()); assert(PPCond[PPStartCond.back()].second == 0); PPCond[PPStartCond.back()].second = index; PPStartCond.pop_back(); // Now add '#elif' as a new block opening. PPCond.push_back(std::make_pair(HashOff, 0U)); PPStartCond.push_back(index); break; } } } EmitToken(Tok); } while (Tok.isNot(tok::eof)); assert(PPStartCond.empty() && "Error: imblanced preprocessor conditionals."); // Next write out PPCond. Offset PPCondOff = (Offset) Out.tell(); // Write out the size of PPCond so that clients can identifer empty tables. Emit32(PPCond.size()); for (unsigned i = 0, e = PPCond.size(); i!=e; ++i) { Emit32(PPCond[i].first - TokenOff); uint32_t x = PPCond[i].second; assert(x != 0 && "PPCond entry not backpatched."); // Emit zero for #endifs. This allows us to do checking when // we read the PTH file back in. Emit32(x == i ? 0 : x); } return PTHEntry(TokenOff, PPCondOff); } Offset PTHWriter::EmitCachedSpellings() { // Write each cached strings to the PTH file. Offset SpellingsOff = Out.tell(); for (std::vector*>::iterator I = StrEntries.begin(), E = StrEntries.end(); I!=E; ++I) EmitBuf((*I)->getKeyData(), (*I)->getKeyLength()+1 /*nul included*/); return SpellingsOff; } void PTHWriter::GeneratePTH(const std::string &MainFile) { // Generate the prologue. Out << "cfe-pth" << '\0'; Emit32(PTHManager::Version); // Leave 4 words for the prologue. Offset PrologueOffset = Out.tell(); for (unsigned i = 0; i < 4; ++i) Emit32(0); // Write the name of the MainFile. if (!MainFile.empty()) { EmitString(MainFile); } else { // String with 0 bytes. Emit16(0); } Emit8(0); // Iterate over all the files in SourceManager. Create a lexer // for each file and cache the tokens. SourceManager &SM = PP.getSourceManager(); const LangOptions &LOpts = PP.getLangOpts(); for (SourceManager::fileinfo_iterator I = SM.fileinfo_begin(), E = SM.fileinfo_end(); I != E; ++I) { const SrcMgr::ContentCache &C = *I->second; const FileEntry *FE = C.OrigEntry; // FIXME: Handle files with non-absolute paths. if (llvm::sys::path::is_relative(FE->getName())) continue; const llvm::MemoryBuffer *B = C.getBuffer(PP.getDiagnostics(), SM); if (!B) continue; FileID FID = SM.createFileID(FE, SourceLocation(), SrcMgr::C_User); const llvm::MemoryBuffer *FromFile = SM.getBuffer(FID); Lexer L(FID, FromFile, SM, LOpts); PM.insert(FE, LexTokens(L)); } // Write out the identifier table. const std::pair &IdTableOff = EmitIdentifierTable(); // Write out the cached strings table. Offset SpellingOff = EmitCachedSpellings(); // Write out the file table. Offset FileTableOff = EmitFileTable(); // Finally, write the prologue. Out.seek(PrologueOffset); Emit32(IdTableOff.first); Emit32(IdTableOff.second); Emit32(FileTableOff); Emit32(SpellingOff); } namespace { /// StatListener - A simple "interpose" object used to monitor stat calls /// invoked by FileManager while processing the original sources used /// as input to PTH generation. StatListener populates the PTHWriter's /// file map with stat information for directories as well as negative stats. /// Stat information for files are populated elsewhere. class StatListener : public FileSystemStatCache { PTHMap &PM; public: StatListener(PTHMap &pm) : PM(pm) {} ~StatListener() {} LookupResult getStat(const char *Path, FileData &Data, bool isFile, int *FileDescriptor) { LookupResult Result = statChained(Path, Data, isFile, FileDescriptor); if (Result == CacheMissing) // Failed 'stat'. PM.insert(PTHEntryKeyVariant(Path), PTHEntry()); else if (Data.IsDirectory) { // Only cache directories with absolute paths. if (llvm::sys::path::is_relative(Path)) return Result; PM.insert(PTHEntryKeyVariant(&Data, Path), PTHEntry()); } return Result; } }; } // end anonymous namespace void clang::CacheTokens(Preprocessor &PP, llvm::raw_fd_ostream* OS) { // Get the name of the main file. const SourceManager &SrcMgr = PP.getSourceManager(); const FileEntry *MainFile = SrcMgr.getFileEntryForID(SrcMgr.getMainFileID()); SmallString<128> MainFilePath(MainFile->getName()); llvm::sys::fs::make_absolute(MainFilePath); // Create the PTHWriter. PTHWriter PW(*OS, PP); // Install the 'stat' system call listener in the FileManager. StatListener *StatCache = new StatListener(PW.getPM()); PP.getFileManager().addStatCache(StatCache, /*AtBeginning=*/true); // Lex through the entire file. This will populate SourceManager with // all of the header information. Token Tok; PP.EnterMainSourceFile(); do { PP.Lex(Tok); } while (Tok.isNot(tok::eof)); // Generate the PTH file. PP.getFileManager().removeStatCache(StatCache); PW.GeneratePTH(MainFilePath.str()); } //===----------------------------------------------------------------------===// namespace { class PTHIdKey { public: const IdentifierInfo* II; uint32_t FileOffset; }; class PTHIdentifierTableTrait { public: typedef PTHIdKey* key_type; typedef key_type key_type_ref; typedef uint32_t data_type; typedef data_type data_type_ref; static unsigned ComputeHash(PTHIdKey* key) { return llvm::HashString(key->II->getName()); } static std::pair EmitKeyDataLength(raw_ostream& Out, const PTHIdKey* key, uint32_t) { unsigned n = key->II->getLength() + 1; ::Emit16(Out, n); return std::make_pair(n, sizeof(uint32_t)); } static void EmitKey(raw_ostream& Out, PTHIdKey* key, unsigned n) { // Record the location of the key data. This is used when generating // the mapping from persistent IDs to strings. key->FileOffset = Out.tell(); Out.write(key->II->getNameStart(), n); } static void EmitData(raw_ostream& Out, PTHIdKey*, uint32_t pID, unsigned) { ::Emit32(Out, pID); } }; } // end anonymous namespace /// EmitIdentifierTable - Emits two tables to the PTH file. The first is /// a hashtable mapping from identifier strings to persistent IDs. The second /// is a straight table mapping from persistent IDs to string data (the /// keys of the first table). /// std::pair PTHWriter::EmitIdentifierTable() { // Build two maps: // (1) an inverse map from persistent IDs -> (IdentifierInfo*,Offset) // (2) a map from (IdentifierInfo*, Offset)* -> persistent IDs // Note that we use 'calloc', so all the bytes are 0. PTHIdKey *IIDMap = (PTHIdKey*)calloc(idcount, sizeof(PTHIdKey)); // Create the hashtable. OnDiskChainedHashTableGenerator IIOffMap; // Generate mapping from persistent IDs -> IdentifierInfo*. for (IDMap::iterator I = IM.begin(), E = IM.end(); I != E; ++I) { // Decrement by 1 because we are using a vector for the lookup and // 0 is reserved for NULL. assert(I->second > 0); assert(I->second-1 < idcount); unsigned idx = I->second-1; // Store the mapping from persistent ID to IdentifierInfo* IIDMap[idx].II = I->first; // Store the reverse mapping in a hashtable. IIOffMap.insert(&IIDMap[idx], I->second); } // Write out the inverse map first. This causes the PCIDKey entries to // record PTH file offsets for the string data. This is used to write // the second table. Offset StringTableOffset = IIOffMap.Emit(Out); // Now emit the table mapping from persistent IDs to PTH file offsets. Offset IDOff = Out.tell(); Emit32(idcount); // Emit the number of identifiers. for (unsigned i = 0 ; i < idcount; ++i) Emit32(IIDMap[i].FileOffset); // Finally, release the inverse map. free(IIDMap); return std::make_pair(IDOff, StringTableOffset); } @ 1.1.1.1 log @Import Clang 3.4rc1 r195771. @ text @@ 1.1.1.2 log @Import Clang 3.5svn r202566. @ text @d519 2 a520 2 vfs::File **F, vfs::FileSystem &FS) { LookupResult Result = statChained(Path, Data, isFile, F, FS); @ 1.1.1.2.2.1 log @Rebase. @ text @d20 1 a25 1 #include "llvm/Support/EndianStream.h" a27 1 #include "llvm/Support/OnDiskHashTable.h" d37 1 a42 2 typedef uint32_t Offset; d64 1 a64 1 PTHEntryKeyVariant(const FileEntry *fe) : FE(fe), Kind(IsFE), Data(nullptr) {} d70 1 a70 1 : Path(path), Kind(IsNoExist), Data(nullptr) {} a80 2 using namespace llvm::support; endian::Writer LE(Out); d85 4 a88 4 LE.write(UID.getFile()); LE.write(UID.getDevice()); LE.write(FE->getModificationTime()); LE.write(FE->getSize()); d92 4 a95 4 LE.write(Data->UniqueID.getFile()); LE.write(Data->UniqueID.getDevice()); LE.write(Data->ModTime); LE.write(Data->Size); d116 1 a116 4 typedef unsigned hash_value_type; typedef unsigned offset_type; static hash_value_type ComputeHash(PTHEntryKeyVariant V) { a122 2 using namespace llvm::support; endian::Writer LE(Out); d125 1 a125 1 LE.write(n); d128 1 a128 1 LE.write(m); a133 1 using namespace llvm::support; d135 1 a135 1 endian::Writer(Out).write((unsigned)V.getKind()); d142 1 a142 2 using namespace llvm::support; endian::Writer LE(Out); d147 2 a148 2 LE.write(E.getTokenOffset()); LE.write(E.getPPCondTableOffset()); d167 1 a167 1 typedef llvm::OnDiskChainedHashTableGenerator PTHMap; d189 1 a189 4 void Emit8(uint32_t V) { using namespace llvm::support; endian::Writer(Out).write(V); } d191 1 a191 4 void Emit16(uint32_t V) { using namespace llvm::support; endian::Writer(Out).write(V); } d193 1 a193 4 void Emit32(uint32_t V) { using namespace llvm::support; endian::Writer(Out).write(V); } d200 1 a200 2 using namespace llvm::support; endian::Writer(Out).write(V.size()); d273 2 a274 5 using namespace llvm::support; endian::Writer LE(Out); uint32_t TokenOff = Out.tell(); for (uint64_t N = llvm::OffsetToAlignment(TokenOff, 4); N; --N, ++TokenOff) LE.write(0); d296 1 a296 1 Tmp.setIdentifierInfo(nullptr); d519 1 a519 1 vfs::File **F, vfs::FileSystem &FS) override { d581 1 a581 4 typedef unsigned hash_value_type; typedef unsigned offset_type; static hash_value_type ComputeHash(PTHIdKey* key) { a586 1 using namespace llvm::support; d588 1 a588 1 endian::Writer(Out).write(n); d601 1 a601 2 using namespace llvm::support; endian::Writer(Out).write(pID); d620 1 a620 1 llvm::OnDiskChainedHashTableGenerator IIOffMap; @ 1.1.1.3 log @Import Clang 3.5svn r209886. @ text @d20 1 a25 1 #include "llvm/Support/EndianStream.h" a27 1 #include "llvm/Support/OnDiskHashTable.h" d37 1 a42 2 typedef uint32_t Offset; d64 1 a64 1 PTHEntryKeyVariant(const FileEntry *fe) : FE(fe), Kind(IsFE), Data(nullptr) {} d70 1 a70 1 : Path(path), Kind(IsNoExist), Data(nullptr) {} a80 2 using namespace llvm::support; endian::Writer LE(Out); d85 4 a88 4 LE.write(UID.getFile()); LE.write(UID.getDevice()); LE.write(FE->getModificationTime()); LE.write(FE->getSize()); d92 4 a95 4 LE.write(Data->UniqueID.getFile()); LE.write(Data->UniqueID.getDevice()); LE.write(Data->ModTime); LE.write(Data->Size); d116 1 a116 4 typedef unsigned hash_value_type; typedef unsigned offset_type; static hash_value_type ComputeHash(PTHEntryKeyVariant V) { a122 2 using namespace llvm::support; endian::Writer LE(Out); d125 1 a125 1 LE.write(n); d128 1 a128 1 LE.write(m); a133 1 using namespace llvm::support; d135 1 a135 1 endian::Writer(Out).write((unsigned)V.getKind()); d142 1 a142 2 using namespace llvm::support; endian::Writer LE(Out); d147 2 a148 2 LE.write(E.getTokenOffset()); LE.write(E.getPPCondTableOffset()); d167 1 a167 1 typedef llvm::OnDiskChainedHashTableGenerator PTHMap; d189 1 a189 4 void Emit8(uint32_t V) { using namespace llvm::support; endian::Writer(Out).write(V); } d191 1 a191 4 void Emit16(uint32_t V) { using namespace llvm::support; endian::Writer(Out).write(V); } d193 1 a193 4 void Emit32(uint32_t V) { using namespace llvm::support; endian::Writer(Out).write(V); } d200 1 a200 2 using namespace llvm::support; endian::Writer(Out).write(V.size()); d273 2 a274 5 using namespace llvm::support; endian::Writer LE(Out); uint32_t TokenOff = Out.tell(); for (uint64_t N = llvm::OffsetToAlignment(TokenOff, 4); N; --N, ++TokenOff) LE.write(0); d296 1 a296 1 Tmp.setIdentifierInfo(nullptr); d519 1 a519 1 vfs::File **F, vfs::FileSystem &FS) override { d581 1 a581 4 typedef unsigned hash_value_type; typedef unsigned offset_type; static hash_value_type ComputeHash(PTHIdKey* key) { a586 1 using namespace llvm::support; d588 1 a588 1 endian::Writer(Out).write(n); d601 1 a601 2 using namespace llvm::support; endian::Writer(Out).write(pID); d620 1 a620 1 llvm::OnDiskChainedHashTableGenerator IIOffMap; @ 1.1.1.4 log @Import clang 3.6svn r215315. @ text @d543 1 a543 2 std::unique_ptr *F, vfs::FileSystem &FS) override { @ 1.1.1.4.2.1 log @Update LLVM to 3.6.1, requested by joerg in ticket 824. @ text @d273 1 a273 1 auto &E = *CachedStrs.insert(std::make_pair(s, OffsetOpt())).first; d276 3 a278 3 if (!E.second.hasOffset()) { E.second.setOffset(CurStrOffset); StrEntries.push_back(&E); d283 1 a283 1 Emit32(E.second.getOffset()); d575 2 a576 4 auto StatCacheOwner = llvm::make_unique(PW.getPM()); StatListener *StatCache = StatCacheOwner.get(); PP.getFileManager().addStatCache(std::move(StatCacheOwner), /*AtBeginning=*/true); @ 1.1.1.5 log @Import Clang 3.6RC1 r227398. @ text @d273 1 a273 1 auto &E = *CachedStrs.insert(std::make_pair(s, OffsetOpt())).first; d276 3 a278 3 if (!E.second.hasOffset()) { E.second.setOffset(CurStrOffset); StrEntries.push_back(&E); d283 1 a283 1 Emit32(E.second.getOffset()); d575 2 a576 4 auto StatCacheOwner = llvm::make_unique(PW.getPM()); StatListener *StatCache = StatCacheOwner.get(); PP.getFileManager().addStatCache(std::move(StatCacheOwner), /*AtBeginning=*/true); @ 1.1.1.6 log @Import Clang 3.8.0rc3 r261930. @ text @a21 1 #include "clang/Lex/PTHManager.h" d108 1 a108 1 return Kind == IsNoExist ? 0 : 4 * 8; d186 1 a186 1 raw_pwrite_stream &Out; d239 2 a240 2 PTHWriter(raw_pwrite_stream &out, Preprocessor &pp) : Out(out), PP(pp), idcount(0), CurStrOffset(0) {} a470 10 static uint32_t swap32le(uint32_t X) { return llvm::support::endian::byte_swap(X); } static void pwrite32le(raw_pwrite_stream &OS, uint32_t Val, uint64_t &Off) { uint32_t LEVal = swap32le(Val); OS.pwrite(reinterpret_cast(&LEVal), 4, Off); Off += 4; } d523 5 a527 5 uint64_t Off = PrologueOffset; pwrite32le(Out, IdTableOff.first, Off); pwrite32le(Out, IdTableOff.second, Off); pwrite32le(Out, FileTableOff, Off); pwrite32le(Out, SpellingOff, Off); d540 1 a540 1 ~StatListener() override {} d562 2 a563 1 void clang::CacheTokens(Preprocessor &PP, raw_pwrite_stream *OS) { @ 1.1.1.6.2.1 log @Sync with HEAD @ text @d15 1 a20 1 #include "clang/Frontend/Utils.h" d31 1 d62 1 a62 6 union { const FileEntry *FE; // FIXME: Use "StringRef Path;" when MSVC 2013 is dropped. const char *PathPtr; }; size_t PathSize; d69 5 a73 7 PTHEntryKeyVariant(FileData *Data, StringRef Path) : PathPtr(Path.data()), PathSize(Path.size()), Kind(IsDE), Data(new FileData(*Data)) {} explicit PTHEntryKeyVariant(StringRef Path) : PathPtr(Path.data()), PathSize(Path.size()), Kind(IsNoExist), Data(nullptr) {} d78 1 a78 1 return Kind == IsFE ? FE->getName() : StringRef(PathPtr, PathSize); d186 1 d189 1 a189 2 IDMap IM; std::vector*> StrEntries; a191 1 uint32_t idcount; d193 1 d244 1 a244 1 void GeneratePTH(StringRef MainFile); d482 1 a482 1 void PTHWriter::GeneratePTH(StringRef MainFile) { d553 1 a553 1 LookupResult getStat(StringRef Path, FileData &Data, bool isFile, @ 1.1.1.7 log @Import Clang pre-4.0.0 r291444. @ text @d15 1 a20 1 #include "clang/Frontend/Utils.h" d31 1 d62 1 a62 6 union { const FileEntry *FE; // FIXME: Use "StringRef Path;" when MSVC 2013 is dropped. const char *PathPtr; }; size_t PathSize; d69 5 a73 7 PTHEntryKeyVariant(FileData *Data, StringRef Path) : PathPtr(Path.data()), PathSize(Path.size()), Kind(IsDE), Data(new FileData(*Data)) {} explicit PTHEntryKeyVariant(StringRef Path) : PathPtr(Path.data()), PathSize(Path.size()), Kind(IsNoExist), Data(nullptr) {} d78 1 a78 1 return Kind == IsFE ? FE->getName() : StringRef(PathPtr, PathSize); d186 1 d189 1 a189 2 IDMap IM; std::vector*> StrEntries; a191 1 uint32_t idcount; d193 1 d244 1 a244 1 void GeneratePTH(StringRef MainFile); d482 1 a482 1 void PTHWriter::GeneratePTH(StringRef MainFile) { d553 1 a553 1 LookupResult getStat(StringRef Path, FileData &Data, bool isFile, @ 1.1.1.7.14.1 log @Sync with HEAD @ text @d24 1 a25 1 #include "llvm/Support/DJB.h" d91 1 a91 1 endian::Writer LE(Out, little); d131 1 a131 1 return llvm::djbHash(V.getString()); d138 1 a138 1 endian::Writer LE(Out, little); d152 1 a152 1 Out << char(V.getKind()); d160 1 a160 1 endian::Writer LE(Out, little); d208 2 a209 1 Out << char(V); d214 1 a214 1 endian::write(Out, V, little); d219 1 a219 1 endian::write(Out, V, little); d228 1 a228 1 endian::write(Out, V.size(), little); d302 1 a302 1 endian::Writer LE(Out, little); d628 1 a628 1 return llvm::djbHash(key->II->getName()); d635 1 a635 1 endian::write(Out, n, little); d649 1 a649 1 endian::write(Out, pID, little); d665 1 a665 2 PTHIdKey *IIDMap = static_cast( llvm::safe_calloc(idcount, sizeof(PTHIdKey))); @ 1.1.1.7.14.2 log @Mostly merge changes from HEAD upto 20200411 @ text @@ 1.1.1.7.12.1 log @Sync with HEAD @ text @d24 1 a25 1 #include "llvm/Support/DJB.h" d91 1 a91 1 endian::Writer LE(Out, little); d131 1 a131 1 return llvm::djbHash(V.getString()); d138 1 a138 1 endian::Writer LE(Out, little); d152 1 a152 1 Out << char(V.getKind()); d160 1 a160 1 endian::Writer LE(Out, little); d208 2 a209 1 Out << char(V); d214 1 a214 1 endian::write(Out, V, little); d219 1 a219 1 endian::write(Out, V, little); d228 1 a228 1 endian::write(Out, V.size(), little); d302 1 a302 1 endian::Writer LE(Out, little); d628 1 a628 1 return llvm::djbHash(key->II->getName()); d635 1 a635 1 endian::write(Out, n, little); d649 1 a649 1 endian::write(Out, pID, little); d665 1 a665 2 PTHIdKey *IIDMap = static_cast( llvm::safe_calloc(idcount, sizeof(PTHIdKey))); @ 1.1.1.8 log @Import clang r337282 from trunk @ text @d24 1 a25 1 #include "llvm/Support/DJB.h" d91 1 a91 1 endian::Writer LE(Out, little); d131 1 a131 1 return llvm::djbHash(V.getString()); d138 1 a138 1 endian::Writer LE(Out, little); d152 1 a152 1 Out << char(V.getKind()); d160 1 a160 1 endian::Writer LE(Out, little); d208 2 a209 1 Out << char(V); d214 1 a214 1 endian::write(Out, V, little); d219 1 a219 1 endian::write(Out, V, little); d228 1 a228 1 endian::write(Out, V.size(), little); d302 1 a302 1 endian::Writer LE(Out, little); d628 1 a628 1 return llvm::djbHash(key->II->getName()); d635 1 a635 1 endian::write(Out, n, little); d649 1 a649 1 endian::write(Out, pID, little); d665 1 a665 2 PTHIdKey *IIDMap = static_cast( llvm::safe_calloc(idcount, sizeof(PTHIdKey))); @ 1.1.1.9 log @Mark old LLVM instance as dead. @ text @@ 1.1.1.4.4.1 log @file CacheTokens.cpp was added on branch tls-maxphys on 2014-08-19 23:47:28 +0000 @ text @d1 682 @ 1.1.1.4.4.2 log @Rebase to HEAD as of a few days ago. @ text @a0 682 //===--- CacheTokens.cpp - Caching of lexer tokens for PTH support --------===// // // The LLVM Compiler Infrastructure // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// // // This provides a possible implementation of PTH support for Clang that is // based on caching lexed tokens and identifiers. // //===----------------------------------------------------------------------===// #include "clang/Frontend/Utils.h" #include "clang/Basic/Diagnostic.h" #include "clang/Basic/FileManager.h" #include "clang/Basic/FileSystemStatCache.h" #include "clang/Basic/IdentifierTable.h" #include "clang/Basic/SourceManager.h" #include "clang/Lex/Lexer.h" #include "clang/Lex/Preprocessor.h" #include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringMap.h" #include "llvm/Support/EndianStream.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/OnDiskHashTable.h" #include "llvm/Support/Path.h" #include "llvm/Support/raw_ostream.h" // FIXME: put this somewhere else? #ifndef S_ISDIR #define S_ISDIR(x) (((x)&_S_IFDIR)!=0) #endif using namespace clang; //===----------------------------------------------------------------------===// // PTH-specific stuff. //===----------------------------------------------------------------------===// typedef uint32_t Offset; namespace { class PTHEntry { Offset TokenData, PPCondData; public: PTHEntry() {} PTHEntry(Offset td, Offset ppcd) : TokenData(td), PPCondData(ppcd) {} Offset getTokenOffset() const { return TokenData; } Offset getPPCondTableOffset() const { return PPCondData; } }; class PTHEntryKeyVariant { union { const FileEntry* FE; const char* Path; }; enum { IsFE = 0x1, IsDE = 0x2, IsNoExist = 0x0 } Kind; FileData *Data; public: PTHEntryKeyVariant(const FileEntry *fe) : FE(fe), Kind(IsFE), Data(nullptr) {} PTHEntryKeyVariant(FileData *Data, const char *path) : Path(path), Kind(IsDE), Data(new FileData(*Data)) {} explicit PTHEntryKeyVariant(const char *path) : Path(path), Kind(IsNoExist), Data(nullptr) {} bool isFile() const { return Kind == IsFE; } StringRef getString() const { return Kind == IsFE ? FE->getName() : Path; } unsigned getKind() const { return (unsigned) Kind; } void EmitData(raw_ostream& Out) { using namespace llvm::support; endian::Writer LE(Out); switch (Kind) { case IsFE: { // Emit stat information. llvm::sys::fs::UniqueID UID = FE->getUniqueID(); LE.write(UID.getFile()); LE.write(UID.getDevice()); LE.write(FE->getModificationTime()); LE.write(FE->getSize()); } break; case IsDE: // Emit stat information. LE.write(Data->UniqueID.getFile()); LE.write(Data->UniqueID.getDevice()); LE.write(Data->ModTime); LE.write(Data->Size); delete Data; break; default: break; } } unsigned getRepresentationLength() const { return Kind == IsNoExist ? 0 : 4 + 4 + 2 + 8 + 8; } }; class FileEntryPTHEntryInfo { public: typedef PTHEntryKeyVariant key_type; typedef key_type key_type_ref; typedef PTHEntry data_type; typedef const PTHEntry& data_type_ref; typedef unsigned hash_value_type; typedef unsigned offset_type; static hash_value_type ComputeHash(PTHEntryKeyVariant V) { return llvm::HashString(V.getString()); } static std::pair EmitKeyDataLength(raw_ostream& Out, PTHEntryKeyVariant V, const PTHEntry& E) { using namespace llvm::support; endian::Writer LE(Out); unsigned n = V.getString().size() + 1 + 1; LE.write(n); unsigned m = V.getRepresentationLength() + (V.isFile() ? 4 + 4 : 0); LE.write(m); return std::make_pair(n, m); } static void EmitKey(raw_ostream& Out, PTHEntryKeyVariant V, unsigned n){ using namespace llvm::support; // Emit the entry kind. endian::Writer(Out).write((unsigned)V.getKind()); // Emit the string. Out.write(V.getString().data(), n - 1); } static void EmitData(raw_ostream& Out, PTHEntryKeyVariant V, const PTHEntry& E, unsigned) { using namespace llvm::support; endian::Writer LE(Out); // For file entries emit the offsets into the PTH file for token data // and the preprocessor blocks table. if (V.isFile()) { LE.write(E.getTokenOffset()); LE.write(E.getPPCondTableOffset()); } // Emit any other data associated with the key (i.e., stat information). V.EmitData(Out); } }; class OffsetOpt { bool valid; Offset off; public: OffsetOpt() : valid(false) {} bool hasOffset() const { return valid; } Offset getOffset() const { assert(valid); return off; } void setOffset(Offset o) { off = o; valid = true; } }; } // end anonymous namespace typedef llvm::OnDiskChainedHashTableGenerator PTHMap; namespace { class PTHWriter { typedef llvm::DenseMap IDMap; typedef llvm::StringMap CachedStrsTy; IDMap IM; llvm::raw_fd_ostream& Out; Preprocessor& PP; uint32_t idcount; PTHMap PM; CachedStrsTy CachedStrs; Offset CurStrOffset; std::vector*> StrEntries; //// Get the persistent id for the given IdentifierInfo*. uint32_t ResolveID(const IdentifierInfo* II); /// Emit a token to the PTH file. void EmitToken(const Token& T); void Emit8(uint32_t V) { using namespace llvm::support; endian::Writer(Out).write(V); } void Emit16(uint32_t V) { using namespace llvm::support; endian::Writer(Out).write(V); } void Emit32(uint32_t V) { using namespace llvm::support; endian::Writer(Out).write(V); } void EmitBuf(const char *Ptr, unsigned NumBytes) { Out.write(Ptr, NumBytes); } void EmitString(StringRef V) { using namespace llvm::support; endian::Writer(Out).write(V.size()); EmitBuf(V.data(), V.size()); } /// EmitIdentifierTable - Emits two tables to the PTH file. The first is /// a hashtable mapping from identifier strings to persistent IDs. /// The second is a straight table mapping from persistent IDs to string data /// (the keys of the first table). std::pair EmitIdentifierTable(); /// EmitFileTable - Emit a table mapping from file name strings to PTH /// token data. Offset EmitFileTable() { return PM.Emit(Out); } PTHEntry LexTokens(Lexer& L); Offset EmitCachedSpellings(); public: PTHWriter(llvm::raw_fd_ostream& out, Preprocessor& pp) : Out(out), PP(pp), idcount(0), CurStrOffset(0) {} PTHMap &getPM() { return PM; } void GeneratePTH(const std::string &MainFile); }; } // end anonymous namespace uint32_t PTHWriter::ResolveID(const IdentifierInfo* II) { // Null IdentifierInfo's map to the persistent ID 0. if (!II) return 0; IDMap::iterator I = IM.find(II); if (I != IM.end()) return I->second; // We've already added 1. IM[II] = ++idcount; // Pre-increment since '0' is reserved for NULL. return idcount; } void PTHWriter::EmitToken(const Token& T) { // Emit the token kind, flags, and length. Emit32(((uint32_t) T.getKind()) | ((((uint32_t) T.getFlags())) << 8)| (((uint32_t) T.getLength()) << 16)); if (!T.isLiteral()) { Emit32(ResolveID(T.getIdentifierInfo())); } else { // We cache *un-cleaned* spellings. This gives us 100% fidelity with the // source code. StringRef s(T.getLiteralData(), T.getLength()); // Get the string entry. llvm::StringMapEntry *E = &CachedStrs.GetOrCreateValue(s); // If this is a new string entry, bump the PTH offset. if (!E->getValue().hasOffset()) { E->getValue().setOffset(CurStrOffset); StrEntries.push_back(E); CurStrOffset += s.size() + 1; } // Emit the relative offset into the PTH file for the spelling string. Emit32(E->getValue().getOffset()); } // Emit the offset into the original source file of this token so that we // can reconstruct its SourceLocation. Emit32(PP.getSourceManager().getFileOffset(T.getLocation())); } PTHEntry PTHWriter::LexTokens(Lexer& L) { // Pad 0's so that we emit tokens to a 4-byte alignment. // This speed up reading them back in. using namespace llvm::support; endian::Writer LE(Out); uint32_t TokenOff = Out.tell(); for (uint64_t N = llvm::OffsetToAlignment(TokenOff, 4); N; --N, ++TokenOff) LE.write(0); // Keep track of matching '#if' ... '#endif'. typedef std::vector > PPCondTable; PPCondTable PPCond; std::vector PPStartCond; bool ParsingPreprocessorDirective = false; Token Tok; do { L.LexFromRawLexer(Tok); NextToken: if ((Tok.isAtStartOfLine() || Tok.is(tok::eof)) && ParsingPreprocessorDirective) { // Insert an eod token into the token cache. It has the same // position as the next token that is not on the same line as the // preprocessor directive. Observe that we continue processing // 'Tok' when we exit this branch. Token Tmp = Tok; Tmp.setKind(tok::eod); Tmp.clearFlag(Token::StartOfLine); Tmp.setIdentifierInfo(nullptr); EmitToken(Tmp); ParsingPreprocessorDirective = false; } if (Tok.is(tok::raw_identifier)) { PP.LookUpIdentifierInfo(Tok); EmitToken(Tok); continue; } if (Tok.is(tok::hash) && Tok.isAtStartOfLine()) { // Special processing for #include. Store the '#' token and lex // the next token. assert(!ParsingPreprocessorDirective); Offset HashOff = (Offset) Out.tell(); // Get the next token. Token NextTok; L.LexFromRawLexer(NextTok); // If we see the start of line, then we had a null directive "#". In // this case, discard both tokens. if (NextTok.isAtStartOfLine()) goto NextToken; // The token is the start of a directive. Emit it. EmitToken(Tok); Tok = NextTok; // Did we see 'include'/'import'/'include_next'? if (Tok.isNot(tok::raw_identifier)) { EmitToken(Tok); continue; } IdentifierInfo* II = PP.LookUpIdentifierInfo(Tok); tok::PPKeywordKind K = II->getPPKeywordID(); ParsingPreprocessorDirective = true; switch (K) { case tok::pp_not_keyword: // Invalid directives "#foo" can occur in #if 0 blocks etc, just pass // them through. default: break; case tok::pp_include: case tok::pp_import: case tok::pp_include_next: { // Save the 'include' token. EmitToken(Tok); // Lex the next token as an include string. L.setParsingPreprocessorDirective(true); L.LexIncludeFilename(Tok); L.setParsingPreprocessorDirective(false); assert(!Tok.isAtStartOfLine()); if (Tok.is(tok::raw_identifier)) PP.LookUpIdentifierInfo(Tok); break; } case tok::pp_if: case tok::pp_ifdef: case tok::pp_ifndef: { // Add an entry for '#if' and friends. We initially set the target // index to 0. This will get backpatched when we hit #endif. PPStartCond.push_back(PPCond.size()); PPCond.push_back(std::make_pair(HashOff, 0U)); break; } case tok::pp_endif: { // Add an entry for '#endif'. We set the target table index to itself. // This will later be set to zero when emitting to the PTH file. We // use 0 for uninitialized indices because that is easier to debug. unsigned index = PPCond.size(); // Backpatch the opening '#if' entry. assert(!PPStartCond.empty()); assert(PPCond.size() > PPStartCond.back()); assert(PPCond[PPStartCond.back()].second == 0); PPCond[PPStartCond.back()].second = index; PPStartCond.pop_back(); // Add the new entry to PPCond. PPCond.push_back(std::make_pair(HashOff, index)); EmitToken(Tok); // Some files have gibberish on the same line as '#endif'. // Discard these tokens. do L.LexFromRawLexer(Tok); while (Tok.isNot(tok::eof) && !Tok.isAtStartOfLine()); // We have the next token in hand. // Don't immediately lex the next one. goto NextToken; } case tok::pp_elif: case tok::pp_else: { // Add an entry for #elif or #else. // This serves as both a closing and opening of a conditional block. // This means that its entry will get backpatched later. unsigned index = PPCond.size(); // Backpatch the previous '#if' entry. assert(!PPStartCond.empty()); assert(PPCond.size() > PPStartCond.back()); assert(PPCond[PPStartCond.back()].second == 0); PPCond[PPStartCond.back()].second = index; PPStartCond.pop_back(); // Now add '#elif' as a new block opening. PPCond.push_back(std::make_pair(HashOff, 0U)); PPStartCond.push_back(index); break; } } } EmitToken(Tok); } while (Tok.isNot(tok::eof)); assert(PPStartCond.empty() && "Error: imblanced preprocessor conditionals."); // Next write out PPCond. Offset PPCondOff = (Offset) Out.tell(); // Write out the size of PPCond so that clients can identifer empty tables. Emit32(PPCond.size()); for (unsigned i = 0, e = PPCond.size(); i!=e; ++i) { Emit32(PPCond[i].first - TokenOff); uint32_t x = PPCond[i].second; assert(x != 0 && "PPCond entry not backpatched."); // Emit zero for #endifs. This allows us to do checking when // we read the PTH file back in. Emit32(x == i ? 0 : x); } return PTHEntry(TokenOff, PPCondOff); } Offset PTHWriter::EmitCachedSpellings() { // Write each cached strings to the PTH file. Offset SpellingsOff = Out.tell(); for (std::vector*>::iterator I = StrEntries.begin(), E = StrEntries.end(); I!=E; ++I) EmitBuf((*I)->getKeyData(), (*I)->getKeyLength()+1 /*nul included*/); return SpellingsOff; } void PTHWriter::GeneratePTH(const std::string &MainFile) { // Generate the prologue. Out << "cfe-pth" << '\0'; Emit32(PTHManager::Version); // Leave 4 words for the prologue. Offset PrologueOffset = Out.tell(); for (unsigned i = 0; i < 4; ++i) Emit32(0); // Write the name of the MainFile. if (!MainFile.empty()) { EmitString(MainFile); } else { // String with 0 bytes. Emit16(0); } Emit8(0); // Iterate over all the files in SourceManager. Create a lexer // for each file and cache the tokens. SourceManager &SM = PP.getSourceManager(); const LangOptions &LOpts = PP.getLangOpts(); for (SourceManager::fileinfo_iterator I = SM.fileinfo_begin(), E = SM.fileinfo_end(); I != E; ++I) { const SrcMgr::ContentCache &C = *I->second; const FileEntry *FE = C.OrigEntry; // FIXME: Handle files with non-absolute paths. if (llvm::sys::path::is_relative(FE->getName())) continue; const llvm::MemoryBuffer *B = C.getBuffer(PP.getDiagnostics(), SM); if (!B) continue; FileID FID = SM.createFileID(FE, SourceLocation(), SrcMgr::C_User); const llvm::MemoryBuffer *FromFile = SM.getBuffer(FID); Lexer L(FID, FromFile, SM, LOpts); PM.insert(FE, LexTokens(L)); } // Write out the identifier table. const std::pair &IdTableOff = EmitIdentifierTable(); // Write out the cached strings table. Offset SpellingOff = EmitCachedSpellings(); // Write out the file table. Offset FileTableOff = EmitFileTable(); // Finally, write the prologue. Out.seek(PrologueOffset); Emit32(IdTableOff.first); Emit32(IdTableOff.second); Emit32(FileTableOff); Emit32(SpellingOff); } namespace { /// StatListener - A simple "interpose" object used to monitor stat calls /// invoked by FileManager while processing the original sources used /// as input to PTH generation. StatListener populates the PTHWriter's /// file map with stat information for directories as well as negative stats. /// Stat information for files are populated elsewhere. class StatListener : public FileSystemStatCache { PTHMap &PM; public: StatListener(PTHMap &pm) : PM(pm) {} ~StatListener() {} LookupResult getStat(const char *Path, FileData &Data, bool isFile, std::unique_ptr *F, vfs::FileSystem &FS) override { LookupResult Result = statChained(Path, Data, isFile, F, FS); if (Result == CacheMissing) // Failed 'stat'. PM.insert(PTHEntryKeyVariant(Path), PTHEntry()); else if (Data.IsDirectory) { // Only cache directories with absolute paths. if (llvm::sys::path::is_relative(Path)) return Result; PM.insert(PTHEntryKeyVariant(&Data, Path), PTHEntry()); } return Result; } }; } // end anonymous namespace void clang::CacheTokens(Preprocessor &PP, llvm::raw_fd_ostream* OS) { // Get the name of the main file. const SourceManager &SrcMgr = PP.getSourceManager(); const FileEntry *MainFile = SrcMgr.getFileEntryForID(SrcMgr.getMainFileID()); SmallString<128> MainFilePath(MainFile->getName()); llvm::sys::fs::make_absolute(MainFilePath); // Create the PTHWriter. PTHWriter PW(*OS, PP); // Install the 'stat' system call listener in the FileManager. StatListener *StatCache = new StatListener(PW.getPM()); PP.getFileManager().addStatCache(StatCache, /*AtBeginning=*/true); // Lex through the entire file. This will populate SourceManager with // all of the header information. Token Tok; PP.EnterMainSourceFile(); do { PP.Lex(Tok); } while (Tok.isNot(tok::eof)); // Generate the PTH file. PP.getFileManager().removeStatCache(StatCache); PW.GeneratePTH(MainFilePath.str()); } //===----------------------------------------------------------------------===// namespace { class PTHIdKey { public: const IdentifierInfo* II; uint32_t FileOffset; }; class PTHIdentifierTableTrait { public: typedef PTHIdKey* key_type; typedef key_type key_type_ref; typedef uint32_t data_type; typedef data_type data_type_ref; typedef unsigned hash_value_type; typedef unsigned offset_type; static hash_value_type ComputeHash(PTHIdKey* key) { return llvm::HashString(key->II->getName()); } static std::pair EmitKeyDataLength(raw_ostream& Out, const PTHIdKey* key, uint32_t) { using namespace llvm::support; unsigned n = key->II->getLength() + 1; endian::Writer(Out).write(n); return std::make_pair(n, sizeof(uint32_t)); } static void EmitKey(raw_ostream& Out, PTHIdKey* key, unsigned n) { // Record the location of the key data. This is used when generating // the mapping from persistent IDs to strings. key->FileOffset = Out.tell(); Out.write(key->II->getNameStart(), n); } static void EmitData(raw_ostream& Out, PTHIdKey*, uint32_t pID, unsigned) { using namespace llvm::support; endian::Writer(Out).write(pID); } }; } // end anonymous namespace /// EmitIdentifierTable - Emits two tables to the PTH file. The first is /// a hashtable mapping from identifier strings to persistent IDs. The second /// is a straight table mapping from persistent IDs to string data (the /// keys of the first table). /// std::pair PTHWriter::EmitIdentifierTable() { // Build two maps: // (1) an inverse map from persistent IDs -> (IdentifierInfo*,Offset) // (2) a map from (IdentifierInfo*, Offset)* -> persistent IDs // Note that we use 'calloc', so all the bytes are 0. PTHIdKey *IIDMap = (PTHIdKey*)calloc(idcount, sizeof(PTHIdKey)); // Create the hashtable. llvm::OnDiskChainedHashTableGenerator IIOffMap; // Generate mapping from persistent IDs -> IdentifierInfo*. for (IDMap::iterator I = IM.begin(), E = IM.end(); I != E; ++I) { // Decrement by 1 because we are using a vector for the lookup and // 0 is reserved for NULL. assert(I->second > 0); assert(I->second-1 < idcount); unsigned idx = I->second-1; // Store the mapping from persistent ID to IdentifierInfo* IIDMap[idx].II = I->first; // Store the reverse mapping in a hashtable. IIOffMap.insert(&IIDMap[idx], I->second); } // Write out the inverse map first. This causes the PCIDKey entries to // record PTH file offsets for the string data. This is used to write // the second table. Offset StringTableOffset = IIOffMap.Emit(Out); // Now emit the table mapping from persistent IDs to PTH file offsets. Offset IDOff = Out.tell(); Emit32(idcount); // Emit the number of identifiers. for (unsigned i = 0 ; i < idcount; ++i) Emit32(IIDMap[i].FileOffset); // Finally, release the inverse map. free(IIDMap); return std::make_pair(IDOff, StringTableOffset); } @ 1.1.1.2.4.1 log @file CacheTokens.cpp was added on branch yamt-pagecache on 2014-05-22 16:18:27 +0000 @ text @d1 652 @ 1.1.1.2.4.2 log @sync with head. for a reference, the tree before this commit was tagged as yamt-pagecache-tag8. this commit was splitted into small chunks to avoid a limitation of cvs. ("Protocol error: too many arguments") @ text @a0 652 //===--- CacheTokens.cpp - Caching of lexer tokens for PTH support --------===// // // The LLVM Compiler Infrastructure // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// // // This provides a possible implementation of PTH support for Clang that is // based on caching lexed tokens and identifiers. // //===----------------------------------------------------------------------===// #include "clang/Frontend/Utils.h" #include "clang/Basic/Diagnostic.h" #include "clang/Basic/FileManager.h" #include "clang/Basic/FileSystemStatCache.h" #include "clang/Basic/IdentifierTable.h" #include "clang/Basic/OnDiskHashTable.h" #include "clang/Basic/SourceManager.h" #include "clang/Lex/Lexer.h" #include "clang/Lex/Preprocessor.h" #include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringMap.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Path.h" #include "llvm/Support/raw_ostream.h" // FIXME: put this somewhere else? #ifndef S_ISDIR #define S_ISDIR(x) (((x)&_S_IFDIR)!=0) #endif using namespace clang; using namespace clang::io; //===----------------------------------------------------------------------===// // PTH-specific stuff. //===----------------------------------------------------------------------===// namespace { class PTHEntry { Offset TokenData, PPCondData; public: PTHEntry() {} PTHEntry(Offset td, Offset ppcd) : TokenData(td), PPCondData(ppcd) {} Offset getTokenOffset() const { return TokenData; } Offset getPPCondTableOffset() const { return PPCondData; } }; class PTHEntryKeyVariant { union { const FileEntry* FE; const char* Path; }; enum { IsFE = 0x1, IsDE = 0x2, IsNoExist = 0x0 } Kind; FileData *Data; public: PTHEntryKeyVariant(const FileEntry *fe) : FE(fe), Kind(IsFE), Data(0) {} PTHEntryKeyVariant(FileData *Data, const char *path) : Path(path), Kind(IsDE), Data(new FileData(*Data)) {} explicit PTHEntryKeyVariant(const char *path) : Path(path), Kind(IsNoExist), Data(0) {} bool isFile() const { return Kind == IsFE; } StringRef getString() const { return Kind == IsFE ? FE->getName() : Path; } unsigned getKind() const { return (unsigned) Kind; } void EmitData(raw_ostream& Out) { switch (Kind) { case IsFE: { // Emit stat information. llvm::sys::fs::UniqueID UID = FE->getUniqueID(); ::Emit64(Out, UID.getFile()); ::Emit64(Out, UID.getDevice()); ::Emit64(Out, FE->getModificationTime()); ::Emit64(Out, FE->getSize()); } break; case IsDE: // Emit stat information. ::Emit64(Out, Data->UniqueID.getFile()); ::Emit64(Out, Data->UniqueID.getDevice()); ::Emit64(Out, Data->ModTime); ::Emit64(Out, Data->Size); delete Data; break; default: break; } } unsigned getRepresentationLength() const { return Kind == IsNoExist ? 0 : 4 + 4 + 2 + 8 + 8; } }; class FileEntryPTHEntryInfo { public: typedef PTHEntryKeyVariant key_type; typedef key_type key_type_ref; typedef PTHEntry data_type; typedef const PTHEntry& data_type_ref; static unsigned ComputeHash(PTHEntryKeyVariant V) { return llvm::HashString(V.getString()); } static std::pair EmitKeyDataLength(raw_ostream& Out, PTHEntryKeyVariant V, const PTHEntry& E) { unsigned n = V.getString().size() + 1 + 1; ::Emit16(Out, n); unsigned m = V.getRepresentationLength() + (V.isFile() ? 4 + 4 : 0); ::Emit8(Out, m); return std::make_pair(n, m); } static void EmitKey(raw_ostream& Out, PTHEntryKeyVariant V, unsigned n){ // Emit the entry kind. ::Emit8(Out, (unsigned) V.getKind()); // Emit the string. Out.write(V.getString().data(), n - 1); } static void EmitData(raw_ostream& Out, PTHEntryKeyVariant V, const PTHEntry& E, unsigned) { // For file entries emit the offsets into the PTH file for token data // and the preprocessor blocks table. if (V.isFile()) { ::Emit32(Out, E.getTokenOffset()); ::Emit32(Out, E.getPPCondTableOffset()); } // Emit any other data associated with the key (i.e., stat information). V.EmitData(Out); } }; class OffsetOpt { bool valid; Offset off; public: OffsetOpt() : valid(false) {} bool hasOffset() const { return valid; } Offset getOffset() const { assert(valid); return off; } void setOffset(Offset o) { off = o; valid = true; } }; } // end anonymous namespace typedef OnDiskChainedHashTableGenerator PTHMap; namespace { class PTHWriter { typedef llvm::DenseMap IDMap; typedef llvm::StringMap CachedStrsTy; IDMap IM; llvm::raw_fd_ostream& Out; Preprocessor& PP; uint32_t idcount; PTHMap PM; CachedStrsTy CachedStrs; Offset CurStrOffset; std::vector*> StrEntries; //// Get the persistent id for the given IdentifierInfo*. uint32_t ResolveID(const IdentifierInfo* II); /// Emit a token to the PTH file. void EmitToken(const Token& T); void Emit8(uint32_t V) { ::Emit8(Out, V); } void Emit16(uint32_t V) { ::Emit16(Out, V); } void Emit32(uint32_t V) { ::Emit32(Out, V); } void EmitBuf(const char *Ptr, unsigned NumBytes) { Out.write(Ptr, NumBytes); } void EmitString(StringRef V) { ::Emit16(Out, V.size()); EmitBuf(V.data(), V.size()); } /// EmitIdentifierTable - Emits two tables to the PTH file. The first is /// a hashtable mapping from identifier strings to persistent IDs. /// The second is a straight table mapping from persistent IDs to string data /// (the keys of the first table). std::pair EmitIdentifierTable(); /// EmitFileTable - Emit a table mapping from file name strings to PTH /// token data. Offset EmitFileTable() { return PM.Emit(Out); } PTHEntry LexTokens(Lexer& L); Offset EmitCachedSpellings(); public: PTHWriter(llvm::raw_fd_ostream& out, Preprocessor& pp) : Out(out), PP(pp), idcount(0), CurStrOffset(0) {} PTHMap &getPM() { return PM; } void GeneratePTH(const std::string &MainFile); }; } // end anonymous namespace uint32_t PTHWriter::ResolveID(const IdentifierInfo* II) { // Null IdentifierInfo's map to the persistent ID 0. if (!II) return 0; IDMap::iterator I = IM.find(II); if (I != IM.end()) return I->second; // We've already added 1. IM[II] = ++idcount; // Pre-increment since '0' is reserved for NULL. return idcount; } void PTHWriter::EmitToken(const Token& T) { // Emit the token kind, flags, and length. Emit32(((uint32_t) T.getKind()) | ((((uint32_t) T.getFlags())) << 8)| (((uint32_t) T.getLength()) << 16)); if (!T.isLiteral()) { Emit32(ResolveID(T.getIdentifierInfo())); } else { // We cache *un-cleaned* spellings. This gives us 100% fidelity with the // source code. StringRef s(T.getLiteralData(), T.getLength()); // Get the string entry. llvm::StringMapEntry *E = &CachedStrs.GetOrCreateValue(s); // If this is a new string entry, bump the PTH offset. if (!E->getValue().hasOffset()) { E->getValue().setOffset(CurStrOffset); StrEntries.push_back(E); CurStrOffset += s.size() + 1; } // Emit the relative offset into the PTH file for the spelling string. Emit32(E->getValue().getOffset()); } // Emit the offset into the original source file of this token so that we // can reconstruct its SourceLocation. Emit32(PP.getSourceManager().getFileOffset(T.getLocation())); } PTHEntry PTHWriter::LexTokens(Lexer& L) { // Pad 0's so that we emit tokens to a 4-byte alignment. // This speed up reading them back in. Pad(Out, 4); Offset TokenOff = (Offset) Out.tell(); // Keep track of matching '#if' ... '#endif'. typedef std::vector > PPCondTable; PPCondTable PPCond; std::vector PPStartCond; bool ParsingPreprocessorDirective = false; Token Tok; do { L.LexFromRawLexer(Tok); NextToken: if ((Tok.isAtStartOfLine() || Tok.is(tok::eof)) && ParsingPreprocessorDirective) { // Insert an eod token into the token cache. It has the same // position as the next token that is not on the same line as the // preprocessor directive. Observe that we continue processing // 'Tok' when we exit this branch. Token Tmp = Tok; Tmp.setKind(tok::eod); Tmp.clearFlag(Token::StartOfLine); Tmp.setIdentifierInfo(0); EmitToken(Tmp); ParsingPreprocessorDirective = false; } if (Tok.is(tok::raw_identifier)) { PP.LookUpIdentifierInfo(Tok); EmitToken(Tok); continue; } if (Tok.is(tok::hash) && Tok.isAtStartOfLine()) { // Special processing for #include. Store the '#' token and lex // the next token. assert(!ParsingPreprocessorDirective); Offset HashOff = (Offset) Out.tell(); // Get the next token. Token NextTok; L.LexFromRawLexer(NextTok); // If we see the start of line, then we had a null directive "#". In // this case, discard both tokens. if (NextTok.isAtStartOfLine()) goto NextToken; // The token is the start of a directive. Emit it. EmitToken(Tok); Tok = NextTok; // Did we see 'include'/'import'/'include_next'? if (Tok.isNot(tok::raw_identifier)) { EmitToken(Tok); continue; } IdentifierInfo* II = PP.LookUpIdentifierInfo(Tok); tok::PPKeywordKind K = II->getPPKeywordID(); ParsingPreprocessorDirective = true; switch (K) { case tok::pp_not_keyword: // Invalid directives "#foo" can occur in #if 0 blocks etc, just pass // them through. default: break; case tok::pp_include: case tok::pp_import: case tok::pp_include_next: { // Save the 'include' token. EmitToken(Tok); // Lex the next token as an include string. L.setParsingPreprocessorDirective(true); L.LexIncludeFilename(Tok); L.setParsingPreprocessorDirective(false); assert(!Tok.isAtStartOfLine()); if (Tok.is(tok::raw_identifier)) PP.LookUpIdentifierInfo(Tok); break; } case tok::pp_if: case tok::pp_ifdef: case tok::pp_ifndef: { // Add an entry for '#if' and friends. We initially set the target // index to 0. This will get backpatched when we hit #endif. PPStartCond.push_back(PPCond.size()); PPCond.push_back(std::make_pair(HashOff, 0U)); break; } case tok::pp_endif: { // Add an entry for '#endif'. We set the target table index to itself. // This will later be set to zero when emitting to the PTH file. We // use 0 for uninitialized indices because that is easier to debug. unsigned index = PPCond.size(); // Backpatch the opening '#if' entry. assert(!PPStartCond.empty()); assert(PPCond.size() > PPStartCond.back()); assert(PPCond[PPStartCond.back()].second == 0); PPCond[PPStartCond.back()].second = index; PPStartCond.pop_back(); // Add the new entry to PPCond. PPCond.push_back(std::make_pair(HashOff, index)); EmitToken(Tok); // Some files have gibberish on the same line as '#endif'. // Discard these tokens. do L.LexFromRawLexer(Tok); while (Tok.isNot(tok::eof) && !Tok.isAtStartOfLine()); // We have the next token in hand. // Don't immediately lex the next one. goto NextToken; } case tok::pp_elif: case tok::pp_else: { // Add an entry for #elif or #else. // This serves as both a closing and opening of a conditional block. // This means that its entry will get backpatched later. unsigned index = PPCond.size(); // Backpatch the previous '#if' entry. assert(!PPStartCond.empty()); assert(PPCond.size() > PPStartCond.back()); assert(PPCond[PPStartCond.back()].second == 0); PPCond[PPStartCond.back()].second = index; PPStartCond.pop_back(); // Now add '#elif' as a new block opening. PPCond.push_back(std::make_pair(HashOff, 0U)); PPStartCond.push_back(index); break; } } } EmitToken(Tok); } while (Tok.isNot(tok::eof)); assert(PPStartCond.empty() && "Error: imblanced preprocessor conditionals."); // Next write out PPCond. Offset PPCondOff = (Offset) Out.tell(); // Write out the size of PPCond so that clients can identifer empty tables. Emit32(PPCond.size()); for (unsigned i = 0, e = PPCond.size(); i!=e; ++i) { Emit32(PPCond[i].first - TokenOff); uint32_t x = PPCond[i].second; assert(x != 0 && "PPCond entry not backpatched."); // Emit zero for #endifs. This allows us to do checking when // we read the PTH file back in. Emit32(x == i ? 0 : x); } return PTHEntry(TokenOff, PPCondOff); } Offset PTHWriter::EmitCachedSpellings() { // Write each cached strings to the PTH file. Offset SpellingsOff = Out.tell(); for (std::vector*>::iterator I = StrEntries.begin(), E = StrEntries.end(); I!=E; ++I) EmitBuf((*I)->getKeyData(), (*I)->getKeyLength()+1 /*nul included*/); return SpellingsOff; } void PTHWriter::GeneratePTH(const std::string &MainFile) { // Generate the prologue. Out << "cfe-pth" << '\0'; Emit32(PTHManager::Version); // Leave 4 words for the prologue. Offset PrologueOffset = Out.tell(); for (unsigned i = 0; i < 4; ++i) Emit32(0); // Write the name of the MainFile. if (!MainFile.empty()) { EmitString(MainFile); } else { // String with 0 bytes. Emit16(0); } Emit8(0); // Iterate over all the files in SourceManager. Create a lexer // for each file and cache the tokens. SourceManager &SM = PP.getSourceManager(); const LangOptions &LOpts = PP.getLangOpts(); for (SourceManager::fileinfo_iterator I = SM.fileinfo_begin(), E = SM.fileinfo_end(); I != E; ++I) { const SrcMgr::ContentCache &C = *I->second; const FileEntry *FE = C.OrigEntry; // FIXME: Handle files with non-absolute paths. if (llvm::sys::path::is_relative(FE->getName())) continue; const llvm::MemoryBuffer *B = C.getBuffer(PP.getDiagnostics(), SM); if (!B) continue; FileID FID = SM.createFileID(FE, SourceLocation(), SrcMgr::C_User); const llvm::MemoryBuffer *FromFile = SM.getBuffer(FID); Lexer L(FID, FromFile, SM, LOpts); PM.insert(FE, LexTokens(L)); } // Write out the identifier table. const std::pair &IdTableOff = EmitIdentifierTable(); // Write out the cached strings table. Offset SpellingOff = EmitCachedSpellings(); // Write out the file table. Offset FileTableOff = EmitFileTable(); // Finally, write the prologue. Out.seek(PrologueOffset); Emit32(IdTableOff.first); Emit32(IdTableOff.second); Emit32(FileTableOff); Emit32(SpellingOff); } namespace { /// StatListener - A simple "interpose" object used to monitor stat calls /// invoked by FileManager while processing the original sources used /// as input to PTH generation. StatListener populates the PTHWriter's /// file map with stat information for directories as well as negative stats. /// Stat information for files are populated elsewhere. class StatListener : public FileSystemStatCache { PTHMap &PM; public: StatListener(PTHMap &pm) : PM(pm) {} ~StatListener() {} LookupResult getStat(const char *Path, FileData &Data, bool isFile, vfs::File **F, vfs::FileSystem &FS) { LookupResult Result = statChained(Path, Data, isFile, F, FS); if (Result == CacheMissing) // Failed 'stat'. PM.insert(PTHEntryKeyVariant(Path), PTHEntry()); else if (Data.IsDirectory) { // Only cache directories with absolute paths. if (llvm::sys::path::is_relative(Path)) return Result; PM.insert(PTHEntryKeyVariant(&Data, Path), PTHEntry()); } return Result; } }; } // end anonymous namespace void clang::CacheTokens(Preprocessor &PP, llvm::raw_fd_ostream* OS) { // Get the name of the main file. const SourceManager &SrcMgr = PP.getSourceManager(); const FileEntry *MainFile = SrcMgr.getFileEntryForID(SrcMgr.getMainFileID()); SmallString<128> MainFilePath(MainFile->getName()); llvm::sys::fs::make_absolute(MainFilePath); // Create the PTHWriter. PTHWriter PW(*OS, PP); // Install the 'stat' system call listener in the FileManager. StatListener *StatCache = new StatListener(PW.getPM()); PP.getFileManager().addStatCache(StatCache, /*AtBeginning=*/true); // Lex through the entire file. This will populate SourceManager with // all of the header information. Token Tok; PP.EnterMainSourceFile(); do { PP.Lex(Tok); } while (Tok.isNot(tok::eof)); // Generate the PTH file. PP.getFileManager().removeStatCache(StatCache); PW.GeneratePTH(MainFilePath.str()); } //===----------------------------------------------------------------------===// namespace { class PTHIdKey { public: const IdentifierInfo* II; uint32_t FileOffset; }; class PTHIdentifierTableTrait { public: typedef PTHIdKey* key_type; typedef key_type key_type_ref; typedef uint32_t data_type; typedef data_type data_type_ref; static unsigned ComputeHash(PTHIdKey* key) { return llvm::HashString(key->II->getName()); } static std::pair EmitKeyDataLength(raw_ostream& Out, const PTHIdKey* key, uint32_t) { unsigned n = key->II->getLength() + 1; ::Emit16(Out, n); return std::make_pair(n, sizeof(uint32_t)); } static void EmitKey(raw_ostream& Out, PTHIdKey* key, unsigned n) { // Record the location of the key data. This is used when generating // the mapping from persistent IDs to strings. key->FileOffset = Out.tell(); Out.write(key->II->getNameStart(), n); } static void EmitData(raw_ostream& Out, PTHIdKey*, uint32_t pID, unsigned) { ::Emit32(Out, pID); } }; } // end anonymous namespace /// EmitIdentifierTable - Emits two tables to the PTH file. The first is /// a hashtable mapping from identifier strings to persistent IDs. The second /// is a straight table mapping from persistent IDs to string data (the /// keys of the first table). /// std::pair PTHWriter::EmitIdentifierTable() { // Build two maps: // (1) an inverse map from persistent IDs -> (IdentifierInfo*,Offset) // (2) a map from (IdentifierInfo*, Offset)* -> persistent IDs // Note that we use 'calloc', so all the bytes are 0. PTHIdKey *IIDMap = (PTHIdKey*)calloc(idcount, sizeof(PTHIdKey)); // Create the hashtable. OnDiskChainedHashTableGenerator IIOffMap; // Generate mapping from persistent IDs -> IdentifierInfo*. for (IDMap::iterator I = IM.begin(), E = IM.end(); I != E; ++I) { // Decrement by 1 because we are using a vector for the lookup and // 0 is reserved for NULL. assert(I->second > 0); assert(I->second-1 < idcount); unsigned idx = I->second-1; // Store the mapping from persistent ID to IdentifierInfo* IIDMap[idx].II = I->first; // Store the reverse mapping in a hashtable. IIOffMap.insert(&IIDMap[idx], I->second); } // Write out the inverse map first. This causes the PCIDKey entries to // record PTH file offsets for the string data. This is used to write // the second table. Offset StringTableOffset = IIOffMap.Emit(Out); // Now emit the table mapping from persistent IDs to PTH file offsets. Offset IDOff = Out.tell(); Emit32(idcount); // Emit the number of identifiers. for (unsigned i = 0 ; i < idcount; ++i) Emit32(IIDMap[i].FileOffset); // Finally, release the inverse map. free(IIDMap); return std::make_pair(IDOff, StringTableOffset); } @