Skip to content

Commit 8edca68

Browse files
ChuanqiXu9DanShaders
authored andcommitted
[clangd] [C++20] [Modules] Introduce initial support for C++20 Modules
Alternatives to https://reviews.llvm.org/D153114. Try to address clangd/clangd#1293. See the links for design ideas. We want to have some initial support in clang18. This is the initial support for C++20 Modules in clangd. As suggested by sammccall in https://reviews.llvm.org/D153114, we should minimize the scope of the initial patch to make it easier to review and understand so that every one are in the same page: > Don't attempt any cross-file or cross-version coordination: i.e. don't > try to reuse BMIs between different files, don't try to reuse BMIs > between (preamble) reparses of the same file, don't try to persist the > module graph. Instead, when building a preamble, synchronously scan > for the module graph, build the required PCMs on the single preamble > thread with filenames private to that preamble, and then proceed to > build the preamble. And this patch reflects the above opinions.
1 parent 096d061 commit 8edca68

31 files changed

+1576
-40
lines changed

clang-tools-extra/clangd/CMakeLists.txt

+4
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,10 @@ add_clang_library(clangDaemon
9797
IncludeFixer.cpp
9898
InlayHints.cpp
9999
JSONTransport.cpp
100+
ModuleDependencyScanner.cpp
101+
ModulesBuilder.cpp
100102
PathMapping.cpp
103+
ProjectModules.cpp
101104
Protocol.cpp
102105
Quality.cpp
103106
ParsedAST.cpp
@@ -161,6 +164,7 @@ clang_target_link_libraries(clangDaemon
161164
clangAST
162165
clangASTMatchers
163166
clangBasic
167+
clangDependencyScanning
164168
clangDriver
165169
clangFormat
166170
clangFrontend

clang-tools-extra/clangd/ClangdServer.cpp

+2
Original file line numberDiff line numberDiff line change
@@ -202,6 +202,7 @@ ClangdServer::Options::operator TUScheduler::Options() const {
202202
Opts.UpdateDebounce = UpdateDebounce;
203203
Opts.ContextProvider = ContextProvider;
204204
Opts.PreambleThrottler = PreambleThrottler;
205+
Opts.ExperimentalModulesSupport = ExperimentalModulesSupport;
205206
return Opts;
206207
}
207208

@@ -222,6 +223,7 @@ ClangdServer::ClangdServer(const GlobalCompilationDatabase &CDB,
222223
DirtyFS(std::make_unique<DraftStoreFS>(TFS, DraftMgr)) {
223224
if (Opts.AsyncThreadsCount != 0)
224225
IndexTasks.emplace();
226+
225227
// Pass a callback into `WorkScheduler` to extract symbols from a newly
226228
// parsed file and rebuild the file index synchronously each time an AST
227229
// is parsed.

clang-tools-extra/clangd/ClangdServer.h

+3
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,9 @@ class ClangdServer {
112112
/// This throttler controls which preambles may be built at a given time.
113113
clangd::PreambleThrottler *PreambleThrottler = nullptr;
114114

115+
/// Enable experimental support for modules.
116+
bool ExperimentalModulesSupport = false;
117+
115118
/// If true, ClangdServer builds a dynamic in-memory index for symbols in
116119
/// opened files and uses the index to augment code completion results.
117120
bool BuildDynamicSymbolIndex = false;

clang-tools-extra/clangd/GlobalCompilationDatabase.cpp

+23
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
#include "GlobalCompilationDatabase.h"
1010
#include "Config.h"
1111
#include "FS.h"
12+
#include "ProjectModules.h"
1213
#include "SourceCode.h"
1314
#include "support/Logger.h"
1415
#include "support/Path.h"
@@ -740,6 +741,21 @@ DirectoryBasedGlobalCompilationDatabase::getProjectInfo(PathRef File) const {
740741
return Res->PI;
741742
}
742743

744+
std::shared_ptr<ProjectModules>
745+
DirectoryBasedGlobalCompilationDatabase::getProjectModules(PathRef File) const {
746+
CDBLookupRequest Req;
747+
Req.FileName = File;
748+
Req.ShouldBroadcast = false;
749+
Req.FreshTime = Req.FreshTimeMissing =
750+
std::chrono::steady_clock::time_point::min();
751+
auto Res = lookupCDB(Req);
752+
if (!Res)
753+
return {};
754+
return ProjectModules::create(
755+
ProjectModules::ProjectModulesKind::ScanningAllFiles,
756+
Res->CDB->getAllFiles(), *this, Opts.TFS);
757+
}
758+
743759
OverlayCDB::OverlayCDB(const GlobalCompilationDatabase *Base,
744760
std::vector<std::string> FallbackFlags,
745761
CommandMangler Mangler)
@@ -832,6 +848,13 @@ std::optional<ProjectInfo> DelegatingCDB::getProjectInfo(PathRef File) const {
832848
return Base->getProjectInfo(File);
833849
}
834850

851+
std::shared_ptr<ProjectModules>
852+
DelegatingCDB::getProjectModules(PathRef File) const {
853+
if (!Base)
854+
return nullptr;
855+
return Base->getProjectModules(File);
856+
}
857+
835858
tooling::CompileCommand DelegatingCDB::getFallbackCommand(PathRef File) const {
836859
if (!Base)
837860
return GlobalCompilationDatabase::getFallbackCommand(File);

clang-tools-extra/clangd/GlobalCompilationDatabase.h

+14
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@
2525
namespace clang {
2626
namespace clangd {
2727

28+
class ProjectModules;
29+
2830
struct ProjectInfo {
2931
// The directory in which the compilation database was discovered.
3032
// Empty if directory-based compilation database discovery was not used.
@@ -45,6 +47,12 @@ class GlobalCompilationDatabase {
4547
return std::nullopt;
4648
}
4749

50+
/// Get the modules in the closest project to \p File
51+
virtual std::shared_ptr<ProjectModules>
52+
getProjectModules(PathRef File) const {
53+
return nullptr;
54+
}
55+
4856
/// Makes a guess at how to build a file.
4957
/// The default implementation just runs clang on the file.
5058
/// Clangd should treat the results as unreliable.
@@ -76,6 +84,9 @@ class DelegatingCDB : public GlobalCompilationDatabase {
7684

7785
std::optional<ProjectInfo> getProjectInfo(PathRef File) const override;
7886

87+
std::shared_ptr<ProjectModules>
88+
getProjectModules(PathRef File) const override;
89+
7990
tooling::CompileCommand getFallbackCommand(PathRef File) const override;
8091

8192
bool blockUntilIdle(Deadline D) const override;
@@ -122,6 +133,9 @@ class DirectoryBasedGlobalCompilationDatabase
122133
/// \p File's parents.
123134
std::optional<ProjectInfo> getProjectInfo(PathRef File) const override;
124135

136+
std::shared_ptr<ProjectModules>
137+
getProjectModules(PathRef File) const override;
138+
125139
bool blockUntilIdle(Deadline Timeout) const override;
126140

127141
private:
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
//===---------------- ModuleDependencyScanner.cpp ----------------*- C++-*-===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
9+
#include "ModuleDependencyScanner.h"
10+
#include "support/Logger.h"
11+
12+
namespace clang {
13+
namespace clangd {
14+
15+
std::optional<ModuleDependencyScanner::ModuleDependencyInfo>
16+
ModuleDependencyScanner::scan(PathRef FilePath) {
17+
std::optional<tooling::CompileCommand> Cmd = CDB.getCompileCommand(FilePath);
18+
19+
if (!Cmd)
20+
return std::nullopt;
21+
22+
using namespace clang::tooling::dependencies;
23+
24+
llvm::SmallString<128> FilePathDir(FilePath);
25+
llvm::sys::path::remove_filename(FilePathDir);
26+
DependencyScanningTool ScanningTool(Service, TFS.view(FilePathDir));
27+
28+
llvm::Expected<P1689Rule> ScanningResult =
29+
ScanningTool.getP1689ModuleDependencyFile(*Cmd, Cmd->Directory);
30+
31+
if (auto E = ScanningResult.takeError()) {
32+
log("Scanning modules dependencies for {0} failed: {1}", FilePath,
33+
llvm::toString(std::move(E)));
34+
return std::nullopt;
35+
}
36+
37+
ModuleDependencyInfo Result;
38+
39+
if (ScanningResult->Provides) {
40+
ModuleNameToSource[ScanningResult->Provides->ModuleName] = FilePath;
41+
Result.ModuleName = ScanningResult->Provides->ModuleName;
42+
}
43+
44+
for (auto &Required : ScanningResult->Requires)
45+
Result.RequiredModules.push_back(Required.ModuleName);
46+
47+
return Result;
48+
}
49+
50+
void ModuleDependencyScanner::globalScan(
51+
const std::vector<std::string> &AllFiles) {
52+
for (auto &File : AllFiles)
53+
scan(File);
54+
55+
GlobalScanned = true;
56+
}
57+
58+
PathRef
59+
ModuleDependencyScanner::getSourceForModuleName(StringRef ModuleName) const {
60+
assert(
61+
GlobalScanned &&
62+
"We should only call getSourceForModuleName after calling globalScan()");
63+
64+
if (auto It = ModuleNameToSource.find(ModuleName);
65+
It != ModuleNameToSource.end())
66+
return It->second;
67+
68+
return {};
69+
}
70+
71+
std::vector<std::string>
72+
ModuleDependencyScanner::getRequiredModules(PathRef File) {
73+
auto ScanningResult = scan(File);
74+
if (!ScanningResult)
75+
return {};
76+
77+
return ScanningResult->RequiredModules;
78+
}
79+
80+
} // namespace clangd
81+
} // namespace clang
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
//===-------------- ModuleDependencyScanner.h --------------------*- C++-*-===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
9+
#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_MODULEDEPENDENCYSCANNER_H
10+
#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_MODULEDEPENDENCYSCANNER_H
11+
12+
#include "GlobalCompilationDatabase.h"
13+
#include "support/Path.h"
14+
#include "support/ThreadsafeFS.h"
15+
16+
#include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
17+
#include "clang/Tooling/DependencyScanning/DependencyScanningTool.h"
18+
19+
#include "llvm/ADT/SmallString.h"
20+
#include "llvm/ADT/StringMap.h"
21+
22+
namespace clang {
23+
namespace clangd {
24+
25+
/// A scanner to query the dependency information for C++20 Modules.
26+
///
27+
/// The scanner can scan a single file with `scan(PathRef)` member function
28+
/// or scan the whole project with `globalScan(PathRef)` member function. See
29+
/// the comments of `globalScan` to see the details.
30+
///
31+
/// ModuleDependencyScanner should only be used via ScanningAllProjectModules.
32+
///
33+
/// The ModuleDependencyScanner can get the directly required module name for a
34+
/// specific source file. Also the ModuleDependencyScanner can get the source
35+
/// file declaring a specific module name.
36+
///
37+
/// IMPORTANT NOTE: we assume that every module unit is only declared once in a
38+
/// source file in the project. But the assumption is not strictly true even
39+
/// besides the invalid projects. The language specification requires that every
40+
/// module unit should be unique in a valid program. But a project can contain
41+
/// multiple programs. Then it is valid that we can have multiple source files
42+
/// declaring the same module in a project as long as these source files don't
43+
/// interere with each other.`
44+
class ModuleDependencyScanner {
45+
public:
46+
ModuleDependencyScanner(const GlobalCompilationDatabase &CDB,
47+
const ThreadsafeFS &TFS)
48+
: CDB(CDB), TFS(TFS),
49+
Service(tooling::dependencies::ScanningMode::CanonicalPreprocessing,
50+
tooling::dependencies::ScanningOutputFormat::P1689) {}
51+
52+
// The scanned modules dependency information for a specific source file.
53+
struct ModuleDependencyInfo {
54+
// The name of the module if the file is a module unit.
55+
std::optional<std::string> ModuleName;
56+
// A list of names for the modules that the file directly depends.
57+
std::vector<std::string> RequiredModules;
58+
};
59+
60+
/// Scanning the single file specified by \param FilePath.
61+
/// NOTE: This is only used by unittests for external uses.
62+
std::optional<ModuleDependencyInfo> scan(PathRef FilePath);
63+
64+
/// Scanning every source file in the current project to get the
65+
/// <module-name> to <module-unit-source> map.
66+
/// It looks unefficiency to scan the whole project especially for
67+
/// every version of every file!
68+
/// TODO: We should find an efficient method to get the <module-name>
69+
/// to <module-unit-source> map. We can make it either by providing
70+
/// a global module dependency scanner to monitor every file. Or we
71+
/// can simply require the build systems (or even if the end users)
72+
/// to provide the map.
73+
void globalScan(const std::vector<std::string> &AllFiles);
74+
bool isGlobalScanned() const { return GlobalScanned; }
75+
76+
/// Get the source file from the module name. Note that the language
77+
/// guarantees all the module names are unique in a valid program.
78+
/// This function should only be called after globalScan.
79+
///
80+
/// FIXME: We should handle the case that there are multiple source files
81+
/// declaring the same module.
82+
PathRef getSourceForModuleName(StringRef ModuleName) const;
83+
84+
/// Return the direct required modules. Indirect required modules are not
85+
/// included.
86+
std::vector<std::string> getRequiredModules(PathRef File);
87+
88+
private:
89+
const GlobalCompilationDatabase &CDB;
90+
const ThreadsafeFS &TFS;
91+
92+
// Whether the scanner has scanned the project globally.
93+
bool GlobalScanned = false;
94+
95+
clang::tooling::dependencies::DependencyScanningService Service;
96+
97+
// TODO: Add a scanning cache.
98+
99+
// Map module name to source file path.
100+
llvm::StringMap<std::string> ModuleNameToSource;
101+
};
102+
103+
} // namespace clangd
104+
} // namespace clang
105+
106+
#endif

0 commit comments

Comments
 (0)