Newer
Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
From ba4c926983036f010c3e4d28be48cfdee8495f87 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Johann=20Kl=C3=A4hn?= <dev@jklaehn.de>
Date: Sun, 9 Jul 2017 13:03:02 +0200
Subject: [PATCH 04/12] Defer addition of keywords to identifier table when
loading AST
In ASTUnit::LoadFromASTFile, the preprocesor object is set up using
default-constructed LangOptions (which only later get populated).
Then, in the constructor of IdentifierTable, these default-constructed
LangOptions were used in the call to AddKeywords, leading to wrong
initialization of the identifier table.
This change defers adding the keywords to the identifier table until
after the language options have been loaded from the AST file.
---
include/clang/Basic/IdentifierTable.h | 6 ++--
include/clang/Lex/Preprocessor.h | 3 +-
lib/Basic/IdentifierTable.cpp | 17 +++++-----
lib/Frontend/ASTUnit.cpp | 10 ++++--
lib/Lex/Preprocessor.cpp | 6 ++--
unittests/libclang/LibclangTest.cpp | 64 +++++++++++++++++++++++++++++++++++
6 files changed, 91 insertions(+), 15 deletions(-)
diff --git a/tools/clang/include/clang/Basic/IdentifierTable.h b/tools/clang/include/clang/Basic/IdentifierTable.h
index 3938e09890..1172869d45 100644
--- a/tools/clang/include/clang/Basic/IdentifierTable.h
+++ b/tools/clang/include/clang/Basic/IdentifierTable.h
@@ -472,9 +472,11 @@ class IdentifierTable {
public:
/// \brief Create the identifier table, populating it with info about the
- /// language keywords for the language specified by \p LangOpts.
+ /// language keywords for the language specified by \p LangOpts if
+ /// \p DeferKeywordAddition is not set.
IdentifierTable(const LangOptions &LangOpts,
- IdentifierInfoLookup* externalLookup = nullptr);
+ IdentifierInfoLookup *externalLookup = nullptr,
+ bool DeferKeywordAddition = false);
/// \brief Set the external identifier lookup mechanism.
void setExternalIdentifierLookup(IdentifierInfoLookup *IILookup) {
diff --git a/tools/clang/include/clang/Lex/Preprocessor.h b/tools/clang/include/clang/Lex/Preprocessor.h
index 49a95986fd..6570e4c76e 100644
--- a/tools/clang/include/clang/Lex/Preprocessor.h
+++ b/tools/clang/include/clang/Lex/Preprocessor.h
@@ -692,7 +692,8 @@ public:
HeaderSearch &Headers, ModuleLoader &TheModuleLoader,
IdentifierInfoLookup *IILookup = nullptr,
bool OwnsHeaderSearch = false,
- TranslationUnitKind TUKind = TU_Complete);
+ TranslationUnitKind TUKind = TU_Complete,
+ bool DeferKeywordAddition = false);
~Preprocessor();
diff --git a/tools/clang/lib/Basic/IdentifierTable.cpp b/tools/clang/lib/Basic/IdentifierTable.cpp
index fe7829ec50..cfb0b1a702 100644
--- a/tools/clang/lib/Basic/IdentifierTable.cpp
+++ b/tools/clang/lib/Basic/IdentifierTable.cpp
@@ -73,17 +73,15 @@ IdentifierIterator *IdentifierInfoLookup::getIdentifiers() {
}
IdentifierTable::IdentifierTable(const LangOptions &LangOpts,
- IdentifierInfoLookup* externalLookup)
- : HashTable(8192), // Start with space for 8K identifiers.
- ExternalLookup(externalLookup) {
+ IdentifierInfoLookup *externalLookup,
+ bool DeferKeywordAddition)
+ : HashTable(8192), // Start with space for 8K identifiers.
+ ExternalLookup(externalLookup) {
// Populate the identifier table with info about keywords for the current
// language.
- AddKeywords(LangOpts);
-
-
- // Add the '_experimental_modules_import' contextual keyword.
- get("import").setModulesImport(true);
+ if (!DeferKeywordAddition)
+ AddKeywords(LangOpts);
}
//===----------------------------------------------------------------------===//
@@ -230,6 +228,9 @@ void IdentifierTable::AddKeywords(const LangOptions &LangOpts) {
if (LangOpts.DeclSpecKeyword)
AddKeyword("__declspec", tok::kw___declspec, KEYALL, LangOpts, *this);
+
+ // Add the '_experimental_modules_import' contextual keyword.
+ get("import").setModulesImport(true);
}
/// \brief Checks if the specified token kind represents a keyword in the
diff --git a/tools/clang/lib/Frontend/ASTUnit.cpp b/tools/clang/lib/Frontend/ASTUnit.cpp
index 07f847ca94..875f21d69a 100644
--- a/tools/clang/lib/Frontend/ASTUnit.cpp
+++ b/tools/clang/lib/Frontend/ASTUnit.cpp
@@ -536,6 +536,10 @@ private:
// Initialize the preprocessor.
PP.Initialize(*Target);
+ // Populate the identifier table with info about keywords for the current
+ // language.
+ PP.getIdentifierTable().AddKeywords(LangOpt);
+
if (!Context)
return;
@@ -718,11 +722,13 @@ std::unique_ptr<ASTUnit> ASTUnit::LoadFromASTFile(
HeaderSearch &HeaderInfo = *AST->HeaderInfo;
unsigned Counter;
+ // As the language options have not been loaded yet, adding keywords to the
+ // identifier table is deferred and will be initiated by ASTInfoCollector.
AST->PP = std::make_shared<Preprocessor>(
AST->PPOpts, AST->getDiagnostics(), *AST->LangOpts,
AST->getSourceManager(), *AST->PCMCache, HeaderInfo, AST->ModuleLoader,
- /*IILookup=*/nullptr,
- /*OwnsHeaderSearch=*/false);
+ /*IILookup=*/nullptr, /*OwnsHeaderSearch=*/false, TU_Complete,
+ /*DeferKeywordAddition=*/true);
Preprocessor &PP = *AST->PP;
if (ToLoad >= LoadASTOnly)
diff --git a/tools/clang/lib/Lex/Preprocessor.cpp b/tools/clang/lib/Lex/Preprocessor.cpp
index 158d0eca27..64365385a3 100644
--- a/tools/clang/lib/Lex/Preprocessor.cpp
+++ b/tools/clang/lib/Lex/Preprocessor.cpp
@@ -73,12 +73,14 @@ Preprocessor::Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts,
SourceManager &SM, MemoryBufferCache &PCMCache,
HeaderSearch &Headers, ModuleLoader &TheModuleLoader,
IdentifierInfoLookup *IILookup, bool OwnsHeaders,
- TranslationUnitKind TUKind)
+ TranslationUnitKind TUKind,
+ bool DeferKeywordAddition)
: PPOpts(std::move(PPOpts)), Diags(&diags), LangOpts(opts), Target(nullptr),
AuxTarget(nullptr), FileMgr(Headers.getFileMgr()), SourceMgr(SM),
PCMCache(PCMCache), ScratchBuf(new ScratchBuffer(SourceMgr)),
HeaderInfo(Headers), TheModuleLoader(TheModuleLoader),
- ExternalSource(nullptr), Identifiers(opts, IILookup),
+ ExternalSource(nullptr),
+ Identifiers(opts, IILookup, DeferKeywordAddition),
PragmaHandlers(new PragmaNamespace(StringRef())),
IncrementalProcessing(false), TUKind(TUKind), CodeComplete(nullptr),
CodeCompletionFile(nullptr), CodeCompletionOffset(0),
diff --git a/tools/clang/unittests/libclang/LibclangTest.cpp b/tools/clang/unittests/libclang/LibclangTest.cpp
index f2a96d6be6..27c8ac7b3d 100644
--- a/tools/clang/unittests/libclang/LibclangTest.cpp
+++ b/tools/clang/unittests/libclang/LibclangTest.cpp
@@ -572,3 +572,67 @@ TEST_F(LibclangReparseTest, clang_parseTranslationUnit2FullArgv) {
EXPECT_EQ(0U, clang_getNumDiagnostics(ClangTU));
DisplayDiagnostics();
}
+
+class LibclangSerializationTest : public LibclangParseTest {
+public:
+ bool SaveAndLoadTU(const std::string &Filename) {
+ unsigned options = clang_defaultSaveOptions(ClangTU);
+ if (clang_saveTranslationUnit(ClangTU, Filename.c_str(), options) !=
+ CXSaveError_None) {
+ DEBUG(llvm::dbgs() << "Saving failed\n");
+ return false;
+ }
+
+ clang_disposeTranslationUnit(ClangTU);
+
+ ClangTU = clang_createTranslationUnit(Index, Filename.c_str());
+
+ if (!ClangTU) {
+ DEBUG(llvm::dbgs() << "Loading failed\n");
+ return false;
+ }
+
+ return true;
+ }
+};
+
+TEST_F(LibclangSerializationTest, TokenKindsAreCorrectAfterLoading) {
+ // Ensure that "class" is recognized as a keyword token after serializing
+ // and reloading the AST, as it is not a keyword for the default LangOptions.
+ std::string HeaderName = "test.h";
+ WriteFile(HeaderName, "enum class Something {};");
+
+ const char *Argv[] = {"-xc++-header", "-std=c++11"};
+
+ ClangTU = clang_parseTranslationUnit(Index, HeaderName.c_str(), Argv,
+ sizeof(Argv) / sizeof(Argv[0]), nullptr,
+ 0, TUFlags);
+
+ auto CheckTokenKinds = [=]() {
+ CXSourceRange Range =
+ clang_getCursorExtent(clang_getTranslationUnitCursor(ClangTU));
+
+ CXToken *Tokens;
+ unsigned int NumTokens;
+ clang_tokenize(ClangTU, Range, &Tokens, &NumTokens);
+
+ ASSERT_EQ(6u, NumTokens);
+ EXPECT_EQ(CXToken_Keyword, clang_getTokenKind(Tokens[0]));
+ EXPECT_EQ(CXToken_Keyword, clang_getTokenKind(Tokens[1]));
+ EXPECT_EQ(CXToken_Identifier, clang_getTokenKind(Tokens[2]));
+ EXPECT_EQ(CXToken_Punctuation, clang_getTokenKind(Tokens[3]));
+ EXPECT_EQ(CXToken_Punctuation, clang_getTokenKind(Tokens[4]));
+ EXPECT_EQ(CXToken_Punctuation, clang_getTokenKind(Tokens[5]));
+
+ clang_disposeTokens(ClangTU, Tokens, NumTokens);
+ };
+
+ CheckTokenKinds();
+
+ std::string ASTName = "test.ast";
+ WriteFile(ASTName, "");
+
+ ASSERT_TRUE(SaveAndLoadTU(ASTName));
+
+ CheckTokenKinds();
+}
--
2.13.0