From 8e3c9361fa143a0cfeaf2819064064399c6f4a21 Mon Sep 17 00:00:00 2001 From: Chuanqi Xu Date: Thu, 18 Jul 2024 10:10:22 +0800 Subject: [PATCH] [clangd] [C++20] [Modules] Introduce initial support for C++20 Modules (#66462) Alternatives to https://reviews.llvm.org/D153114. Try to address https://github.com/clangd/clangd/issues/1293. See the links for design ideas and the consensus so far. We want to have some initial support in clang18. This is the initial support for C++20 Modules in clangd. As suggested by sammccall in https://reviews.llvm.org/D153114, we should minimize the scope of the initial patch to make it easier to review and understand so that every one are in the same page: > Don't attempt any cross-file or cross-version coordination: i.e. don't > try to reuse BMIs between different files, don't try to reuse BMIs > between (preamble) reparses of the same file, don't try to persist the > module graph. Instead, when building a preamble, synchronously scan > for the module graph, build the required PCMs on the single preamble > thread with filenames private to that preamble, and then proceed to > build the preamble. This patch reflects the above opinions. # Testing in real-world project I tested this with a modularized library: https://github.com/alibaba/async_simple/tree/CXX20Modules. This library has 3 modules (async_simple, std and asio) and 65 module units. (Note that a module consists of multiple module units). Both `std` module and `asio` module have 100k+ lines of code (maybe more, I didn't count). And async_simple itself has 8k lines of code. This is the scale of the project. The result shows that it works pretty well, ..., well, except I need to wait roughly 10s after opening/editing any file. And this falls in our expectations. We know it is hard to make it perfect in the first move. # What this patch does in detail - Introduced an option `--experimental-modules-support` for the support for C++20 Modules. So that no matter how bad this is, it wouldn't affect current users. Following off the page, we'll assume the option is enabled. - Introduced two classes `ModuleFilesInfo` and `ModuleDependencyScanner`. Now `ModuleDependencyScanner` is only used by `ModuleFilesInfo`. - The class `ModuleFilesInfo` records the built module files for specific single source file. The module files can only be built by the static member function `ModuleFilesInfo::buildModuleFilesInfoFor(PathRef File, ...)`. - The class `PreambleData` adds a new member variable with type `ModuleFilesInfo`. This refers to the needed module files for the current file. It means the module files info is part of the preamble, which is suggested in the first patch too. - In `isPreambleCompatible()`, we add a call to `ModuleFilesInfo::CanReuse()` to check if the built module files are still up to date. - When we build the AST for a source file, we will load the built module files from ModuleFilesInfo. # What we need to do next Let's split the TODOs into clang part and clangd part to make things more clear. The TODOs in the clangd part include: 1. Enable reusing module files across source files. The may require us to bring a ModulesManager like thing which need to handle `scheduling`, `the possibility of BMI version conflicts` and `various events that can invalidate the module graph`. 2. Get a more efficient method to get the ` -> ` map. Currently we always scan the whole project during `ModuleFilesInfo::buildModuleFilesInfoFor(PathRef File, ...)`. This is clearly inefficient even if the scanning process is pretty fast. I think the potential solutions include: - Make a global scanner to monitor the state of every source file like I did in the first patch. The pain point is that we need to take care of the data races. - Ask the build systems to provide the map just like we ask them to provide the compilation database. 3. Persist the module files. So that we can reuse module files across clangd invocations or even across clangd instances. TODOs in the clang part include: 1. Clang should offer an option/mode to skip writing/reading the bodies of the functions. Or even if we can requrie the parser to skip parsing the function bodies. And it looks like we can say the support for C++20 Modules is initially workable after we made (1) and (2) (or even without (2)). --- clang-tools-extra/clangd/CMakeLists.txt | 3 + clang-tools-extra/clangd/ClangdLSPServer.cpp | 8 + clang-tools-extra/clangd/ClangdLSPServer.h | 5 + clang-tools-extra/clangd/ClangdServer.cpp | 2 + clang-tools-extra/clangd/ClangdServer.h | 6 + clang-tools-extra/clangd/Compiler.h | 3 + .../clangd/GlobalCompilationDatabase.cpp | 23 + .../clangd/GlobalCompilationDatabase.h | 13 + clang-tools-extra/clangd/ModulesBuilder.cpp | 336 +++++++++++++++ clang-tools-extra/clangd/ModulesBuilder.h | 106 +++++ clang-tools-extra/clangd/ParsedAST.cpp | 7 + clang-tools-extra/clangd/Preamble.cpp | 18 +- clang-tools-extra/clangd/Preamble.h | 4 + clang-tools-extra/clangd/ProjectModules.h | 50 +++ .../clangd/ScanningProjectModules.cpp | 202 +++++++++ .../clangd/ScanningProjectModules.h | 26 ++ clang-tools-extra/clangd/test/CMakeLists.txt | 1 + clang-tools-extra/clangd/test/modules.test | 83 ++++ clang-tools-extra/clangd/tool/Check.cpp | 12 +- clang-tools-extra/clangd/tool/ClangdMain.cpp | 8 + .../clangd/unittests/CMakeLists.txt | 1 + .../unittests/PrerequisiteModulesTest.cpp | 408 ++++++++++++++++++ clang-tools-extra/clangd/unittests/TestFS.h | 2 +- clang-tools-extra/docs/ReleaseNotes.rst | 4 + 24 files changed, 1327 insertions(+), 4 deletions(-) create mode 100644 clang-tools-extra/clangd/ModulesBuilder.cpp create mode 100644 clang-tools-extra/clangd/ModulesBuilder.h create mode 100644 clang-tools-extra/clangd/ProjectModules.h create mode 100644 clang-tools-extra/clangd/ScanningProjectModules.cpp create mode 100644 clang-tools-extra/clangd/ScanningProjectModules.h create mode 100644 clang-tools-extra/clangd/test/modules.test create mode 100644 clang-tools-extra/clangd/unittests/PrerequisiteModulesTest.cpp diff --git a/clang-tools-extra/clangd/CMakeLists.txt b/clang-tools-extra/clangd/CMakeLists.txt index f49704157880d3a..c21d277d2ffcbd8 100644 --- a/clang-tools-extra/clangd/CMakeLists.txt +++ b/clang-tools-extra/clangd/CMakeLists.txt @@ -97,12 +97,14 @@ add_clang_library(clangDaemon IncludeFixer.cpp InlayHints.cpp JSONTransport.cpp + ModulesBuilder.cpp PathMapping.cpp Protocol.cpp Quality.cpp ParsedAST.cpp Preamble.cpp RIFF.cpp + ScanningProjectModules.cpp Selection.cpp SemanticHighlighting.cpp SemanticSelection.cpp @@ -161,6 +163,7 @@ clang_target_link_libraries(clangDaemon clangAST clangASTMatchers clangBasic + clangDependencyScanning clangDriver clangFormat clangFrontend diff --git a/clang-tools-extra/clangd/ClangdLSPServer.cpp b/clang-tools-extra/clangd/ClangdLSPServer.cpp index 7fd599d4e1a0b0e..06573a575542451 100644 --- a/clang-tools-extra/clangd/ClangdLSPServer.cpp +++ b/clang-tools-extra/clangd/ClangdLSPServer.cpp @@ -14,6 +14,7 @@ #include "Feature.h" #include "GlobalCompilationDatabase.h" #include "LSPBinder.h" +#include "ModulesBuilder.h" #include "Protocol.h" #include "SemanticHighlighting.h" #include "SourceCode.h" @@ -51,6 +52,7 @@ namespace clang { namespace clangd { + namespace { // Tracks end-to-end latency of high level lsp calls. Measurements are in // seconds. @@ -563,6 +565,12 @@ void ClangdLSPServer::onInitialize(const InitializeParams &Params, Mangler.ResourceDir = *Opts.ResourceDir; CDB.emplace(BaseCDB.get(), Params.initializationOptions.fallbackFlags, std::move(Mangler)); + + if (Opts.EnableExperimentalModulesSupport) { + ModulesManager.emplace(*CDB); + Opts.ModulesManager = &*ModulesManager; + } + { // Switch caller's context with LSPServer's background context. Since we // rather want to propagate information from LSPServer's context into the diff --git a/clang-tools-extra/clangd/ClangdLSPServer.h b/clang-tools-extra/clangd/ClangdLSPServer.h index 8bcb29522509b75..0b8e4720f53236b 100644 --- a/clang-tools-extra/clangd/ClangdLSPServer.h +++ b/clang-tools-extra/clangd/ClangdLSPServer.h @@ -63,6 +63,9 @@ class ClangdLSPServer : private ClangdServer::Callbacks, /// Limit the number of references returned (0 means no limit). size_t ReferencesLimit = 0; + + /// Flag to hint the experimental modules support is enabled. + bool EnableExperimentalModulesSupport = false; }; ClangdLSPServer(Transport &Transp, const ThreadsafeFS &TFS, @@ -323,6 +326,8 @@ class ClangdLSPServer : private ClangdServer::Callbacks, std::optional CDB; // The ClangdServer is created by the "initialize" LSP method. std::optional Server; + // Manages to build module files. + std::optional ModulesManager; }; } // namespace clangd } // namespace clang diff --git a/clang-tools-extra/clangd/ClangdServer.cpp b/clang-tools-extra/clangd/ClangdServer.cpp index 1c4c2a79b5c0510..e910a80ba0bae9b 100644 --- a/clang-tools-extra/clangd/ClangdServer.cpp +++ b/clang-tools-extra/clangd/ClangdServer.cpp @@ -216,6 +216,7 @@ ClangdServer::ClangdServer(const GlobalCompilationDatabase &CDB, Callbacks *Callbacks) : FeatureModules(Opts.FeatureModules), CDB(CDB), TFS(TFS), DynamicIdx(Opts.BuildDynamicSymbolIndex ? new FileIndex() : nullptr), + ModulesManager(Opts.ModulesManager), ClangTidyProvider(Opts.ClangTidyProvider), UseDirtyHeaders(Opts.UseDirtyHeaders), LineFoldingOnly(Opts.LineFoldingOnly), @@ -308,6 +309,7 @@ void ClangdServer::addDocument(PathRef File, llvm::StringRef Contents, Inputs.Index = Index; Inputs.ClangTidyProvider = ClangTidyProvider; Inputs.FeatureModules = FeatureModules; + Inputs.ModulesManager = ModulesManager; bool NewFile = WorkScheduler->update(File, Inputs, WantDiags); // If we loaded Foo.h, we want to make sure Foo.cpp is indexed. if (NewFile && BackgroundIdx) diff --git a/clang-tools-extra/clangd/ClangdServer.h b/clang-tools-extra/clangd/ClangdServer.h index 1661028be88b4e7..a653cdb56b751b6 100644 --- a/clang-tools-extra/clangd/ClangdServer.h +++ b/clang-tools-extra/clangd/ClangdServer.h @@ -16,6 +16,7 @@ #include "FeatureModule.h" #include "GlobalCompilationDatabase.h" #include "Hover.h" +#include "ModulesBuilder.h" #include "Protocol.h" #include "SemanticHighlighting.h" #include "TUScheduler.h" @@ -112,6 +113,9 @@ class ClangdServer { /// This throttler controls which preambles may be built at a given time. clangd::PreambleThrottler *PreambleThrottler = nullptr; + /// Manages to build module files. + ModulesBuilder *ModulesManager = nullptr; + /// If true, ClangdServer builds a dynamic in-memory index for symbols in /// opened files and uses the index to augment code completion results. bool BuildDynamicSymbolIndex = false; @@ -477,6 +481,8 @@ class ClangdServer { std::unique_ptr BackgroundIdx; // Storage for merged views of the various indexes. std::vector> MergedIdx; + // Manage module files. + ModulesBuilder *ModulesManager = nullptr; // When set, provides clang-tidy options for a specific file. TidyProviderRef ClangTidyProvider; diff --git a/clang-tools-extra/clangd/Compiler.h b/clang-tools-extra/clangd/Compiler.h index 56c2567ebe97b0c..4e68da7610ca2c3 100644 --- a/clang-tools-extra/clangd/Compiler.h +++ b/clang-tools-extra/clangd/Compiler.h @@ -16,6 +16,7 @@ #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_COMPILER_H #include "FeatureModule.h" +#include "ModulesBuilder.h" #include "TidyProvider.h" #include "index/Index.h" #include "support/ThreadsafeFS.h" @@ -60,6 +61,8 @@ struct ParseInputs { TidyProviderRef ClangTidyProvider = {}; // Used to acquire ASTListeners when parsing files. FeatureModuleSet *FeatureModules = nullptr; + // Used to build and manage (C++) modules. + ModulesBuilder *ModulesManager = nullptr; }; /// Clears \p CI from options that are not supported by clangd, like codegen or diff --git a/clang-tools-extra/clangd/GlobalCompilationDatabase.cpp b/clang-tools-extra/clangd/GlobalCompilationDatabase.cpp index 85c80eb482efbf3..1d96667a8e9f4a7 100644 --- a/clang-tools-extra/clangd/GlobalCompilationDatabase.cpp +++ b/clang-tools-extra/clangd/GlobalCompilationDatabase.cpp @@ -9,6 +9,8 @@ #include "GlobalCompilationDatabase.h" #include "Config.h" #include "FS.h" +#include "ProjectModules.h" +#include "ScanningProjectModules.h" #include "SourceCode.h" #include "support/Logger.h" #include "support/Path.h" @@ -741,6 +743,20 @@ DirectoryBasedGlobalCompilationDatabase::getProjectInfo(PathRef File) const { return Res->PI; } +std::unique_ptr +DirectoryBasedGlobalCompilationDatabase::getProjectModules(PathRef File) const { + CDBLookupRequest Req; + Req.FileName = File; + Req.ShouldBroadcast = false; + Req.FreshTime = Req.FreshTimeMissing = + std::chrono::steady_clock::time_point::min(); + auto Res = lookupCDB(Req); + if (!Res) + return {}; + + return scanningProjectModules(Res->CDB, Opts.TFS); +} + OverlayCDB::OverlayCDB(const GlobalCompilationDatabase *Base, std::vector FallbackFlags, CommandMangler Mangler) @@ -833,6 +849,13 @@ std::optional DelegatingCDB::getProjectInfo(PathRef File) const { return Base->getProjectInfo(File); } +std::unique_ptr +DelegatingCDB::getProjectModules(PathRef File) const { + if (!Base) + return nullptr; + return Base->getProjectModules(File); +} + tooling::CompileCommand DelegatingCDB::getFallbackCommand(PathRef File) const { if (!Base) return GlobalCompilationDatabase::getFallbackCommand(File); diff --git a/clang-tools-extra/clangd/GlobalCompilationDatabase.h b/clang-tools-extra/clangd/GlobalCompilationDatabase.h index 2bf8c973c534c6f..ea999fe8aee0177 100644 --- a/clang-tools-extra/clangd/GlobalCompilationDatabase.h +++ b/clang-tools-extra/clangd/GlobalCompilationDatabase.h @@ -9,6 +9,7 @@ #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_GLOBALCOMPILATIONDATABASE_H #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_GLOBALCOMPILATIONDATABASE_H +#include "ProjectModules.h" #include "support/Function.h" #include "support/Path.h" #include "support/Threading.h" @@ -45,6 +46,12 @@ class GlobalCompilationDatabase { return std::nullopt; } + /// Get the modules in the closest project to \p File + virtual std::unique_ptr + getProjectModules(PathRef File) const { + return nullptr; + } + /// Makes a guess at how to build a file. /// The default implementation just runs clang on the file. /// Clangd should treat the results as unreliable. @@ -76,6 +83,9 @@ class DelegatingCDB : public GlobalCompilationDatabase { std::optional getProjectInfo(PathRef File) const override; + std::unique_ptr + getProjectModules(PathRef File) const override; + tooling::CompileCommand getFallbackCommand(PathRef File) const override; bool blockUntilIdle(Deadline D) const override; @@ -122,6 +132,9 @@ class DirectoryBasedGlobalCompilationDatabase /// \p File's parents. std::optional getProjectInfo(PathRef File) const override; + std::unique_ptr + getProjectModules(PathRef File) const override; + bool blockUntilIdle(Deadline Timeout) const override; private: diff --git a/clang-tools-extra/clangd/ModulesBuilder.cpp b/clang-tools-extra/clangd/ModulesBuilder.cpp new file mode 100644 index 000000000000000..94c7eec2d09e4ee --- /dev/null +++ b/clang-tools-extra/clangd/ModulesBuilder.cpp @@ -0,0 +1,336 @@ +//===----------------- ModulesBuilder.cpp ------------------------*- C++-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "ModulesBuilder.h" +#include "Compiler.h" +#include "support/Logger.h" +#include "clang/Frontend/FrontendAction.h" +#include "clang/Frontend/FrontendActions.h" +#include "clang/Serialization/ASTReader.h" + +namespace clang { +namespace clangd { + +namespace { + +// Create a path to store module files. Generally it should be: +// +// {TEMP_DIRS}/clangd/module_files/{hashed-file-name}-%%-%%-%%-%%-%%-%%/. +// +// {TEMP_DIRS} is the temporary directory for the system, e.g., "/var/tmp" +// or "C:/TEMP". +// +// '%%' means random value to make the generated path unique. +// +// \param MainFile is used to get the root of the project from global +// compilation database. +// +// TODO: Move these module fils out of the temporary directory if the module +// files are persistent. +llvm::SmallString<256> getUniqueModuleFilesPath(PathRef MainFile) { + llvm::SmallString<128> HashedPrefix = llvm::sys::path::filename(MainFile); + // There might be multiple files with the same name in a project. So appending + // the hash value of the full path to make sure they won't conflict. + HashedPrefix += std::to_string(llvm::hash_value(MainFile)); + + llvm::SmallString<256> ResultPattern; + + llvm::sys::path::system_temp_directory(/*erasedOnReboot=*/true, + ResultPattern); + + llvm::sys::path::append(ResultPattern, "clangd"); + llvm::sys::path::append(ResultPattern, "module_files"); + + llvm::sys::path::append(ResultPattern, HashedPrefix); + + ResultPattern.append("-%%-%%-%%-%%-%%-%%"); + + llvm::SmallString<256> Result; + llvm::sys::fs::createUniquePath(ResultPattern, Result, + /*MakeAbsolute=*/false); + + llvm::sys::fs::create_directories(Result); + return Result; +} + +// Get a unique module file path under \param ModuleFilesPrefix. +std::string getModuleFilePath(llvm::StringRef ModuleName, + PathRef ModuleFilesPrefix) { + llvm::SmallString<256> ModuleFilePath(ModuleFilesPrefix); + auto [PrimaryModuleName, PartitionName] = ModuleName.split(':'); + llvm::sys::path::append(ModuleFilePath, PrimaryModuleName); + if (!PartitionName.empty()) { + ModuleFilePath.append("-"); + ModuleFilePath.append(PartitionName); + } + + ModuleFilePath.append(".pcm"); + return std::string(ModuleFilePath); +} + +// FailedPrerequisiteModules - stands for the PrerequisiteModules which has +// errors happened during the building process. +class FailedPrerequisiteModules : public PrerequisiteModules { +public: + ~FailedPrerequisiteModules() override = default; + + // We shouldn't adjust the compilation commands based on + // FailedPrerequisiteModules. + void adjustHeaderSearchOptions(HeaderSearchOptions &Options) const override { + } + + // FailedPrerequisiteModules can never be reused. + bool + canReuse(const CompilerInvocation &CI, + llvm::IntrusiveRefCntPtr) const override { + return false; + } +}; + +// StandalonePrerequisiteModules - stands for PrerequisiteModules for which all +// the required modules are built successfully. All the module files +// are owned by the StandalonePrerequisiteModules class. +// +// Any of the built module files won't be shared with other instances of the +// class. So that we can avoid worrying thread safety. +// +// We don't need to worry about duplicated module names here since the standard +// guarantees the module names should be unique to a program. +class StandalonePrerequisiteModules : public PrerequisiteModules { +public: + StandalonePrerequisiteModules() = default; + + StandalonePrerequisiteModules(const StandalonePrerequisiteModules &) = delete; + StandalonePrerequisiteModules + operator=(const StandalonePrerequisiteModules &) = delete; + StandalonePrerequisiteModules(StandalonePrerequisiteModules &&) = delete; + StandalonePrerequisiteModules + operator=(StandalonePrerequisiteModules &&) = delete; + + ~StandalonePrerequisiteModules() override = default; + + void adjustHeaderSearchOptions(HeaderSearchOptions &Options) const override { + // Appending all built module files. + for (auto &RequiredModule : RequiredModules) + Options.PrebuiltModuleFiles.insert_or_assign( + RequiredModule.ModuleName, RequiredModule.ModuleFilePath); + } + + bool canReuse(const CompilerInvocation &CI, + llvm::IntrusiveRefCntPtr) const override; + + bool isModuleUnitBuilt(llvm::StringRef ModuleName) const { + return BuiltModuleNames.contains(ModuleName); + } + + void addModuleFile(llvm::StringRef ModuleName, + llvm::StringRef ModuleFilePath) { + RequiredModules.emplace_back(ModuleName, ModuleFilePath); + BuiltModuleNames.insert(ModuleName); + } + +private: + struct ModuleFile { + ModuleFile(llvm::StringRef ModuleName, PathRef ModuleFilePath) + : ModuleName(ModuleName.str()), ModuleFilePath(ModuleFilePath.str()) {} + + ModuleFile(const ModuleFile &) = delete; + ModuleFile operator=(const ModuleFile &) = delete; + + // The move constructor is needed for llvm::SmallVector. + ModuleFile(ModuleFile &&Other) + : ModuleName(std::move(Other.ModuleName)), + ModuleFilePath(std::move(Other.ModuleFilePath)) {} + + ModuleFile &operator=(ModuleFile &&Other) = delete; + + ~ModuleFile() { + if (!ModuleFilePath.empty()) + llvm::sys::fs::remove(ModuleFilePath); + } + + std::string ModuleName; + std::string ModuleFilePath; + }; + + llvm::SmallVector RequiredModules; + // A helper class to speedup the query if a module is built. + llvm::StringSet<> BuiltModuleNames; +}; + +// Build a module file for module with `ModuleName`. The information of built +// module file are stored in \param BuiltModuleFiles. +llvm::Error buildModuleFile(llvm::StringRef ModuleName, + const GlobalCompilationDatabase &CDB, + const ThreadsafeFS &TFS, ProjectModules &MDB, + PathRef ModuleFilesPrefix, + StandalonePrerequisiteModules &BuiltModuleFiles) { + if (BuiltModuleFiles.isModuleUnitBuilt(ModuleName)) + return llvm::Error::success(); + + PathRef ModuleUnitFileName = MDB.getSourceForModuleName(ModuleName); + // It is possible that we're meeting third party modules (modules whose + // source are not in the project. e.g, the std module may be a third-party + // module for most projects) or something wrong with the implementation of + // ProjectModules. + // FIXME: How should we treat third party modules here? If we want to ignore + // third party modules, we should return true instead of false here. + // Currently we simply bail out. + if (ModuleUnitFileName.empty()) + return llvm::createStringError("Failed to get the primary source"); + + // Try cheap operation earlier to boil-out cheaply if there are problems. + auto Cmd = CDB.getCompileCommand(ModuleUnitFileName); + if (!Cmd) + return llvm::createStringError( + llvm::formatv("No compile command for {0}", ModuleUnitFileName)); + + for (auto &RequiredModuleName : MDB.getRequiredModules(ModuleUnitFileName)) { + // Return early if there are errors building the module file. + if (llvm::Error Err = buildModuleFile(RequiredModuleName, CDB, TFS, MDB, + ModuleFilesPrefix, BuiltModuleFiles)) + return llvm::createStringError( + llvm::formatv("Failed to build dependency {0}: {1}", + RequiredModuleName, llvm::toString(std::move(Err)))); + } + + Cmd->Output = getModuleFilePath(ModuleName, ModuleFilesPrefix); + + ParseInputs Inputs; + Inputs.TFS = &TFS; + Inputs.CompileCommand = std::move(*Cmd); + + IgnoreDiagnostics IgnoreDiags; + auto CI = buildCompilerInvocation(Inputs, IgnoreDiags); + if (!CI) + return llvm::createStringError("Failed to build compiler invocation"); + + auto FS = Inputs.TFS->view(Inputs.CompileCommand.Directory); + auto Buf = FS->getBufferForFile(Inputs.CompileCommand.Filename); + if (!Buf) + return llvm::createStringError("Failed to create buffer"); + + // In clang's driver, we will suppress the check for ODR violation in GMF. + // See the implementation of RenderModulesOptions in Clang.cpp. + CI->getLangOpts().SkipODRCheckInGMF = true; + + // Hash the contents of input files and store the hash value to the BMI files. + // So that we can check if the files are still valid when we want to reuse the + // BMI files. + CI->getHeaderSearchOpts().ValidateASTInputFilesContent = true; + + BuiltModuleFiles.adjustHeaderSearchOptions(CI->getHeaderSearchOpts()); + + CI->getFrontendOpts().OutputFile = Inputs.CompileCommand.Output; + auto Clang = + prepareCompilerInstance(std::move(CI), /*Preamble=*/nullptr, + std::move(*Buf), std::move(FS), IgnoreDiags); + if (!Clang) + return llvm::createStringError("Failed to prepare compiler instance"); + + GenerateReducedModuleInterfaceAction Action; + Clang->ExecuteAction(Action); + + if (Clang->getDiagnostics().hasErrorOccurred()) + return llvm::createStringError("Compilation failed"); + + BuiltModuleFiles.addModuleFile(ModuleName, Inputs.CompileCommand.Output); + return llvm::Error::success(); +} +} // namespace + +std::unique_ptr +ModulesBuilder::buildPrerequisiteModulesFor(PathRef File, + const ThreadsafeFS &TFS) const { + std::unique_ptr MDB = CDB.getProjectModules(File); + if (!MDB) { + elog("Failed to get Project Modules information for {0}", File); + return std::make_unique(); + } + + std::vector RequiredModuleNames = MDB->getRequiredModules(File); + if (RequiredModuleNames.empty()) + return std::make_unique(); + + llvm::SmallString<256> ModuleFilesPrefix = getUniqueModuleFilesPath(File); + + log("Trying to build required modules for {0} in {1}", File, + ModuleFilesPrefix); + + auto RequiredModules = std::make_unique(); + + for (llvm::StringRef RequiredModuleName : RequiredModuleNames) { + // Return early if there is any error. + if (llvm::Error Err = + buildModuleFile(RequiredModuleName, CDB, TFS, *MDB.get(), + ModuleFilesPrefix, *RequiredModules.get())) { + elog("Failed to build module {0}; due to {1}", RequiredModuleName, + toString(std::move(Err))); + return std::make_unique(); + } + } + + log("Built required modules for {0} in {1}", File, ModuleFilesPrefix); + + return std::move(RequiredModules); +} + +bool StandalonePrerequisiteModules::canReuse( + const CompilerInvocation &CI, + llvm::IntrusiveRefCntPtr VFS) const { + if (RequiredModules.empty()) + return true; + + CompilerInstance Clang; + + Clang.setInvocation(std::make_shared(CI)); + IntrusiveRefCntPtr Diags = + CompilerInstance::createDiagnostics(new DiagnosticOptions()); + Clang.setDiagnostics(Diags.get()); + + FileManager *FM = Clang.createFileManager(VFS); + Clang.createSourceManager(*FM); + + if (!Clang.createTarget()) + return false; + + assert(Clang.getHeaderSearchOptsPtr()); + adjustHeaderSearchOptions(Clang.getHeaderSearchOpts()); + // Since we don't need to compile the source code actually, the TU kind here + // doesn't matter. + Clang.createPreprocessor(TU_Complete); + Clang.getHeaderSearchOpts().ForceCheckCXX20ModulesInputFiles = true; + Clang.getHeaderSearchOpts().ValidateASTInputFilesContent = true; + + // Following the practice of clang's driver to suppres the checking for ODR + // violation in GMF. + // See + // https://clang.llvm.org/docs/StandardCPlusPlusModules.html#object-definition-consistency + // for example. + Clang.getLangOpts().SkipODRCheckInGMF = true; + + Clang.createASTReader(); + for (auto &RequiredModule : RequiredModules) { + llvm::StringRef BMIPath = RequiredModule.ModuleFilePath; + // FIXME: Loading BMI fully is too heavy considering something cheaply to + // check if we can reuse the BMI. + auto ReadResult = + Clang.getASTReader()->ReadAST(BMIPath, serialization::MK_MainFile, + SourceLocation(), ASTReader::ARR_None); + + if (ReadResult != ASTReader::Success) { + elog("Can't reuse {0}: {1}", BMIPath, ReadResult); + return false; + } + } + + return true; +} + +} // namespace clangd +} // namespace clang diff --git a/clang-tools-extra/clangd/ModulesBuilder.h b/clang-tools-extra/clangd/ModulesBuilder.h new file mode 100644 index 000000000000000..0514e7486475d09 --- /dev/null +++ b/clang-tools-extra/clangd/ModulesBuilder.h @@ -0,0 +1,106 @@ +//===----------------- ModulesBuilder.h --------------------------*- C++-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Experimental support for C++20 Modules. +// +// Currently we simplify the implementations by preventing reusing module files +// across different versions and different source files. But this is clearly a +// waste of time and space in the end of the day. +// +// TODO: Supporting reusing module files across different versions and +// different source files. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_MODULES_BUILDER_H +#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_MODULES_BUILDER_H + +#include "GlobalCompilationDatabase.h" +#include "ProjectModules.h" +#include "support/Path.h" +#include "support/ThreadsafeFS.h" +#include "clang/Frontend/CompilerInvocation.h" +#include "llvm/ADT/SmallString.h" +#include + +namespace clang { +namespace clangd { + +/// Store all the needed module files information to parse a single +/// source file. e.g., +/// +/// ``` +/// // a.cppm +/// export module a; +/// +/// // b.cppm +/// export module b; +/// import a; +/// +/// // c.cppm +/// export module c; +/// import b; +/// ``` +/// +/// For the source file `c.cppm`, an instance of the class will store +/// the module files for `a.cppm` and `b.cppm`. But the module file for `c.cppm` +/// won't be stored. Since it is not needed to parse `c.cppm`. +/// +/// Users should only get PrerequisiteModules from +/// `ModulesBuilder::buildPrerequisiteModulesFor(...)`. +/// +/// Users can detect whether the PrerequisiteModules is still up to date by +/// calling the `canReuse()` member function. +/// +/// The users should call `adjustHeaderSearchOptions(...)` to update the +/// compilation commands to select the built module files first. Before calling +/// `adjustHeaderSearchOptions()`, users should call `canReuse()` first to check +/// if all the stored module files are valid. In case they are not valid, +/// users should call `ModulesBuilder::buildPrerequisiteModulesFor(...)` again +/// to get the new PrerequisiteModules. +class PrerequisiteModules { +public: + /// Change commands to load the module files recorded in this + /// PrerequisiteModules first. + virtual void + adjustHeaderSearchOptions(HeaderSearchOptions &Options) const = 0; + + /// Whether or not the built module files are up to date. + /// Note that this can only be used after building the module files. + virtual bool + canReuse(const CompilerInvocation &CI, + llvm::IntrusiveRefCntPtr) const = 0; + + virtual ~PrerequisiteModules() = default; +}; + +/// This class handles building module files for a given source file. +/// +/// In the future, we want the class to manage the module files acorss +/// different versions and different source files. +class ModulesBuilder { +public: + ModulesBuilder(const GlobalCompilationDatabase &CDB) : CDB(CDB) {} + + ModulesBuilder(const ModulesBuilder &) = delete; + ModulesBuilder(ModulesBuilder &&) = delete; + + ModulesBuilder &operator=(const ModulesBuilder &) = delete; + ModulesBuilder &operator=(ModulesBuilder &&) = delete; + + std::unique_ptr + buildPrerequisiteModulesFor(PathRef File, const ThreadsafeFS &TFS) const; + +private: + const GlobalCompilationDatabase &CDB; +}; + +} // namespace clangd +} // namespace clang + +#endif diff --git a/clang-tools-extra/clangd/ParsedAST.cpp b/clang-tools-extra/clangd/ParsedAST.cpp index 2bd1fbcad2ada0f..a2f1504db7e880e 100644 --- a/clang-tools-extra/clangd/ParsedAST.cpp +++ b/clang-tools-extra/clangd/ParsedAST.cpp @@ -446,6 +446,12 @@ ParsedAST::build(llvm::StringRef Filename, const ParseInputs &Inputs, L->sawDiagnostic(D, Diag); }); + // Adjust header search options to load the built module files recorded + // in RequiredModules. + if (Preamble && Preamble->RequiredModules) + Preamble->RequiredModules->adjustHeaderSearchOptions( + CI->getHeaderSearchOpts()); + std::optional Patch; // We might use an ignoring diagnostic consumer if they are going to be // dropped later on to not pay for extra latency by processing them. @@ -459,6 +465,7 @@ ParsedAST::build(llvm::StringRef Filename, const ParseInputs &Inputs, std::move(CI), PreamblePCH, llvm::MemoryBuffer::getMemBufferCopy(Inputs.Contents, Filename), VFS, *DiagConsumer); + if (!Clang) { // The last diagnostic contains information about the reason of this // failure. diff --git a/clang-tools-extra/clangd/Preamble.cpp b/clang-tools-extra/clangd/Preamble.cpp index ecd490145dd3c45..dd13b1a9e5613d2 100644 --- a/clang-tools-extra/clangd/Preamble.cpp +++ b/clang-tools-extra/clangd/Preamble.cpp @@ -664,6 +664,7 @@ buildPreamble(PathRef FileName, CompilerInvocation CI, CI, ContentsBuffer.get(), Bounds, *PreambleDiagsEngine, Stats ? TimedFS : StatCacheFS, std::make_shared(), StoreInMemory, /*StoragePath=*/"", CapturedInfo); + PreambleTimer.stopTimer(); // We have to setup DiagnosticConsumer that will be alife @@ -696,6 +697,19 @@ buildPreamble(PathRef FileName, CompilerInvocation CI, Result->Includes = CapturedInfo.takeIncludes(); Result->Pragmas = std::make_shared( CapturedInfo.takePragmaIncludes()); + + if (Inputs.ModulesManager) { + WallTimer PrerequisiteModuleTimer; + PrerequisiteModuleTimer.startTimer(); + Result->RequiredModules = + Inputs.ModulesManager->buildPrerequisiteModulesFor(FileName, + *Inputs.TFS); + PrerequisiteModuleTimer.stopTimer(); + + log("Built prerequisite modules for file {0} in {1} seconds", FileName, + PrerequisiteModuleTimer.getTime()); + } + Result->Macros = CapturedInfo.takeMacros(); Result->Marks = CapturedInfo.takeMarks(); Result->StatCache = StatCache; @@ -737,7 +751,9 @@ bool isPreambleCompatible(const PreambleData &Preamble, auto VFS = Inputs.TFS->view(Inputs.CompileCommand.Directory); return compileCommandsAreEqual(Inputs.CompileCommand, Preamble.CompileCommand) && - Preamble.Preamble.CanReuse(CI, *ContentsBuffer, Bounds, *VFS); + Preamble.Preamble.CanReuse(CI, *ContentsBuffer, Bounds, *VFS) && + (!Preamble.RequiredModules || + Preamble.RequiredModules->canReuse(CI, VFS)); } void escapeBackslashAndQuotes(llvm::StringRef Text, llvm::raw_ostream &OS) { diff --git a/clang-tools-extra/clangd/Preamble.h b/clang-tools-extra/clangd/Preamble.h index 160b884beb56bb7..be8fed4ab88cdd8 100644 --- a/clang-tools-extra/clangd/Preamble.h +++ b/clang-tools-extra/clangd/Preamble.h @@ -27,6 +27,8 @@ #include "Diagnostics.h" #include "FS.h" #include "Headers.h" +#include "ModulesBuilder.h" + #include "clang-include-cleaner/Record.h" #include "support/Path.h" #include "clang/Basic/SourceManager.h" @@ -109,6 +111,8 @@ struct PreambleData { IncludeStructure Includes; // Captures #include-mapping information in #included headers. std::shared_ptr Pragmas; + // Information about required module files for this preamble. + std::unique_ptr RequiredModules; // Macros defined in the preamble section of the main file. // Users care about headers vs main-file, not preamble vs non-preamble. // These should be treated as main-file entities e.g. for code completion. diff --git a/clang-tools-extra/clangd/ProjectModules.h b/clang-tools-extra/clangd/ProjectModules.h new file mode 100644 index 000000000000000..3b9b564a87da015 --- /dev/null +++ b/clang-tools-extra/clangd/ProjectModules.h @@ -0,0 +1,50 @@ +//===------------------ ProjectModules.h -------------------------*- C++-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_PROJECTMODULES_H +#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_PROJECTMODULES_H + +#include "support/Path.h" +#include "support/ThreadsafeFS.h" + +#include + +namespace clang { +namespace clangd { + +/// An interface to query the modules information in the project. +/// Users should get instances of `ProjectModules` from +/// `GlobalCompilationDatabase::getProjectModules(PathRef)`. +/// +/// Currently, the modules information includes: +/// - Given a source file, what are the required modules. +/// - Given a module name and a required source file, what is +/// the corresponding source file. +/// +/// Note that there can be multiple source files declaring the same module +/// in a valid project. Although the language specification requires that +/// every module unit's name must be unique in valid program, there can be +/// multiple program in a project. And it is technically valid if these program +/// doesn't interfere with each other. +/// +/// A module name should be in the format: +/// `[:partition-name]`. So module names covers partitions. +class ProjectModules { +public: + virtual std::vector getRequiredModules(PathRef File) = 0; + virtual PathRef + getSourceForModuleName(llvm::StringRef ModuleName, + PathRef RequiredSrcFile = PathRef()) = 0; + + virtual ~ProjectModules() = default; +}; + +} // namespace clangd +} // namespace clang + +#endif diff --git a/clang-tools-extra/clangd/ScanningProjectModules.cpp b/clang-tools-extra/clangd/ScanningProjectModules.cpp new file mode 100644 index 000000000000000..92f75ef7d5c25ad --- /dev/null +++ b/clang-tools-extra/clangd/ScanningProjectModules.cpp @@ -0,0 +1,202 @@ +//===------------------ ProjectModules.h -------------------------*- C++-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "ProjectModules.h" +#include "support/Logger.h" +#include "clang/Tooling/DependencyScanning/DependencyScanningService.h" +#include "clang/Tooling/DependencyScanning/DependencyScanningTool.h" + +namespace clang::clangd { +namespace { +/// A scanner to query the dependency information for C++20 Modules. +/// +/// The scanner can scan a single file with `scan(PathRef)` member function +/// or scan the whole project with `globalScan(vector)` member +/// function. See the comments of `globalScan` to see the details. +/// +/// The ModuleDependencyScanner can get the directly required module names for a +/// specific source file. Also the ModuleDependencyScanner can get the source +/// file declaring the primary module interface for a specific module name. +/// +/// IMPORTANT NOTE: we assume that every module unit is only declared once in a +/// source file in the project. But the assumption is not strictly true even +/// besides the invalid projects. The language specification requires that every +/// module unit should be unique in a valid program. But a project can contain +/// multiple programs. Then it is valid that we can have multiple source files +/// declaring the same module in a project as long as these source files don't +/// interfere with each other. +class ModuleDependencyScanner { +public: + ModuleDependencyScanner( + std::shared_ptr CDB, + const ThreadsafeFS &TFS) + : CDB(CDB), TFS(TFS), + Service(tooling::dependencies::ScanningMode::CanonicalPreprocessing, + tooling::dependencies::ScanningOutputFormat::P1689) {} + + /// The scanned modules dependency information for a specific source file. + struct ModuleDependencyInfo { + /// The name of the module if the file is a module unit. + std::optional ModuleName; + /// A list of names for the modules that the file directly depends. + std::vector RequiredModules; + }; + + /// Scanning the single file specified by \param FilePath. + std::optional scan(PathRef FilePath); + + /// Scanning every source file in the current project to get the + /// to map. + /// TODO: We should find an efficient method to get the + /// to map. We can make it either by providing + /// a global module dependency scanner to monitor every file. Or we + /// can simply require the build systems (or even the end users) + /// to provide the map. + void globalScan(); + + /// Get the source file from the module name. Note that the language + /// guarantees all the module names are unique in a valid program. + /// This function should only be called after globalScan. + /// + /// TODO: We should handle the case that there are multiple source files + /// declaring the same module. + PathRef getSourceForModuleName(llvm::StringRef ModuleName) const; + + /// Return the direct required modules. Indirect required modules are not + /// included. + std::vector getRequiredModules(PathRef File); + +private: + std::shared_ptr CDB; + const ThreadsafeFS &TFS; + + // Whether the scanner has scanned the project globally. + bool GlobalScanned = false; + + clang::tooling::dependencies::DependencyScanningService Service; + + // TODO: Add a scanning cache. + + // Map module name to source file path. + llvm::StringMap ModuleNameToSource; +}; + +std::optional +ModuleDependencyScanner::scan(PathRef FilePath) { + auto Candidates = CDB->getCompileCommands(FilePath); + if (Candidates.empty()) + return std::nullopt; + + // Choose the first candidates as the compile commands as the file. + // Following the same logic with + // DirectoryBasedGlobalCompilationDatabase::getCompileCommand. + tooling::CompileCommand Cmd = std::move(Candidates.front()); + + static int StaticForMainAddr; // Just an address in this process. + Cmd.CommandLine.push_back("-resource-dir=" + + CompilerInvocation::GetResourcesPath( + "clangd", (void *)&StaticForMainAddr)); + + using namespace clang::tooling::dependencies; + + llvm::SmallString<128> FilePathDir(FilePath); + llvm::sys::path::remove_filename(FilePathDir); + DependencyScanningTool ScanningTool(Service, TFS.view(FilePathDir)); + + llvm::Expected ScanningResult = + ScanningTool.getP1689ModuleDependencyFile(Cmd, Cmd.Directory); + + if (auto E = ScanningResult.takeError()) { + elog("Scanning modules dependencies for {0} failed: {1}", FilePath, + llvm::toString(std::move(E))); + return std::nullopt; + } + + ModuleDependencyInfo Result; + + if (ScanningResult->Provides) { + ModuleNameToSource[ScanningResult->Provides->ModuleName] = FilePath; + Result.ModuleName = ScanningResult->Provides->ModuleName; + } + + for (auto &Required : ScanningResult->Requires) + Result.RequiredModules.push_back(Required.ModuleName); + + return Result; +} + +void ModuleDependencyScanner::globalScan() { + for (auto &File : CDB->getAllFiles()) + scan(File); + + GlobalScanned = true; +} + +PathRef ModuleDependencyScanner::getSourceForModuleName( + llvm::StringRef ModuleName) const { + assert( + GlobalScanned && + "We should only call getSourceForModuleName after calling globalScan()"); + + if (auto It = ModuleNameToSource.find(ModuleName); + It != ModuleNameToSource.end()) + return It->second; + + return {}; +} + +std::vector +ModuleDependencyScanner::getRequiredModules(PathRef File) { + auto ScanningResult = scan(File); + if (!ScanningResult) + return {}; + + return ScanningResult->RequiredModules; +} +} // namespace + +/// TODO: The existing `ScanningAllProjectModules` is not efficient. See the +/// comments in ModuleDependencyScanner for detail. +/// +/// In the future, we wish the build system can provide a well design +/// compilation database for modules then we can query that new compilation +/// database directly. Or we need to have a global long-live scanner to detect +/// the state of each file. +class ScanningAllProjectModules : public ProjectModules { +public: + ScanningAllProjectModules( + std::shared_ptr CDB, + const ThreadsafeFS &TFS) + : Scanner(CDB, TFS) {} + + ~ScanningAllProjectModules() override = default; + + std::vector getRequiredModules(PathRef File) override { + return Scanner.getRequiredModules(File); + } + + /// RequiredSourceFile is not used intentionally. See the comments of + /// ModuleDependencyScanner for detail. + PathRef + getSourceForModuleName(llvm::StringRef ModuleName, + PathRef RequiredSourceFile = PathRef()) override { + Scanner.globalScan(); + return Scanner.getSourceForModuleName(ModuleName); + } + +private: + ModuleDependencyScanner Scanner; +}; + +std::unique_ptr scanningProjectModules( + std::shared_ptr CDB, + const ThreadsafeFS &TFS) { + return std::make_unique(CDB, TFS); +} + +} // namespace clang::clangd diff --git a/clang-tools-extra/clangd/ScanningProjectModules.h b/clang-tools-extra/clangd/ScanningProjectModules.h new file mode 100644 index 000000000000000..75fc7dbcebce540 --- /dev/null +++ b/clang-tools-extra/clangd/ScanningProjectModules.h @@ -0,0 +1,26 @@ +//===------------ ScanningProjectModules.h -----------------------*- C++-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_SCANNINGPROJECTMODULES_H +#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_SCANNINGPROJECTMODULES_H + +#include "ProjectModules.h" +#include "clang/Tooling/CompilationDatabase.h" + +namespace clang { +namespace clangd { + +/// Providing modules information for the project by scanning every file. +std::unique_ptr scanningProjectModules( + std::shared_ptr CDB, + const ThreadsafeFS &TFS); + +} // namespace clangd +} // namespace clang + +#endif diff --git a/clang-tools-extra/clangd/test/CMakeLists.txt b/clang-tools-extra/clangd/test/CMakeLists.txt index d073267066e0b4c..b51f461a4986659 100644 --- a/clang-tools-extra/clangd/test/CMakeLists.txt +++ b/clang-tools-extra/clangd/test/CMakeLists.txt @@ -2,6 +2,7 @@ set(CLANGD_TEST_DEPS clangd ClangdTests clangd-indexer + split-file # No tests for it, but we should still make sure they build. dexp ) diff --git a/clang-tools-extra/clangd/test/modules.test b/clang-tools-extra/clangd/test/modules.test new file mode 100644 index 000000000000000..74280811a6cff84 --- /dev/null +++ b/clang-tools-extra/clangd/test/modules.test @@ -0,0 +1,83 @@ +# A smoke test to check the modules can work basically. +# +# Windows have different escaping modes. +# FIXME: We should add one for windows. +# UNSUPPORTED: system-windows +# +# RUN: rm -fr %t +# RUN: mkdir -p %t +# RUN: split-file %s %t +# +# RUN: sed -e "s|DIR|%/t|g" %t/compile_commands.json.tmpl > %t/compile_commands.json.tmp +# RUN: sed -e "s|CLANG_CC|%clang|g" %t/compile_commands.json.tmp > %t/compile_commands.json +# RUN: sed -e "s|DIR|%/t|g" %t/definition.jsonrpc.tmpl > %t/definition.jsonrpc +# +# RUN: clangd -experimental-modules-support -lit-test < %t/definition.jsonrpc \ +# RUN: | FileCheck -strict-whitespace %t/definition.jsonrpc + +#--- A.cppm +export module A; +export void printA() {} + +#--- Use.cpp +import A; +void foo() { + print +} + +#--- compile_commands.json.tmpl +[ + { + "directory": "DIR", + "command": "CLANG_CC -fprebuilt-module-path=DIR -std=c++20 -o DIR/main.cpp.o -c DIR/Use.cpp", + "file": "DIR/Use.cpp" + }, + { + "directory": "DIR", + "command": "CLANG_CC -std=c++20 DIR/A.cppm --precompile -o DIR/A.pcm", + "file": "DIR/A.cppm" + } +] + +#--- definition.jsonrpc.tmpl +{ + "jsonrpc": "2.0", + "id": 0, + "method": "initialize", + "params": { + "processId": 123, + "rootPath": "clangd", + "capabilities": { + "textDocument": { + "completion": { + "completionItem": { + "snippetSupport": true + } + } + } + }, + "trace": "off" + } +} +--- +{ + "jsonrpc": "2.0", + "method": "textDocument/didOpen", + "params": { + "textDocument": { + "uri": "file://DIR/Use.cpp", + "languageId": "cpp", + "version": 1, + "text": "import A;\nvoid foo() {\n print\n}\n" + } + } +} + +# CHECK: "message"{{.*}}printA{{.*}}(fix available) + +--- +{"jsonrpc":"2.0","id":1,"method":"textDocument/completion","params":{"textDocument":{"uri":"file://DIR/Use.cpp"},"context":{"triggerKind":1},"position":{"line":2,"character":6}}} +--- +{"jsonrpc":"2.0","id":2,"method":"shutdown"} +--- +{"jsonrpc":"2.0","method":"exit"} diff --git a/clang-tools-extra/clangd/tool/Check.cpp b/clang-tools-extra/clangd/tool/Check.cpp index 25005ec1bd04532..bc2eaa77a66eeca 100644 --- a/clang-tools-extra/clangd/tool/Check.cpp +++ b/clang-tools-extra/clangd/tool/Check.cpp @@ -146,10 +146,13 @@ class Checker { ClangdLSPServer::Options Opts; // from buildCommand tooling::CompileCommand Cmd; + std::unique_ptr BaseCDB; + std::unique_ptr CDB; // from buildInvocation ParseInputs Inputs; std::unique_ptr Invocation; format::FormatStyle Style; + std::optional ModulesManager; // from buildAST std::shared_ptr Preamble; std::optional AST; @@ -168,14 +171,14 @@ class Checker { DirectoryBasedGlobalCompilationDatabase::Options CDBOpts(TFS); CDBOpts.CompileCommandsDir = Config::current().CompileFlags.CDBSearch.FixedCDBPath; - std::unique_ptr BaseCDB = + BaseCDB = std::make_unique(CDBOpts); auto Mangler = CommandMangler::detect(); Mangler.SystemIncludeExtractor = getSystemIncludeExtractor(llvm::ArrayRef(Opts.QueryDriverGlobs)); if (Opts.ResourceDir) Mangler.ResourceDir = *Opts.ResourceDir; - auto CDB = std::make_unique( + CDB = std::make_unique( BaseCDB.get(), std::vector{}, std::move(Mangler)); if (auto TrueCmd = CDB->getCompileCommand(File)) { @@ -213,6 +216,11 @@ class Checker { return false; } } + if (Opts.EnableExperimentalModulesSupport) { + if (!ModulesManager) + ModulesManager.emplace(*CDB); + Inputs.ModulesManager = &*ModulesManager; + } log("Parsing command..."); Invocation = buildCompilerInvocation(Inputs, CaptureInvocationDiags, &CC1Args); diff --git a/clang-tools-extra/clangd/tool/ClangdMain.cpp b/clang-tools-extra/clangd/tool/ClangdMain.cpp index 73000d96c6ca8e5..3a5449ac8c79944 100644 --- a/clang-tools-extra/clangd/tool/ClangdMain.cpp +++ b/clang-tools-extra/clangd/tool/ClangdMain.cpp @@ -551,6 +551,13 @@ opt ProjectRoot{ }; #endif +opt ExperimentalModulesSupport{ + "experimental-modules-support", + cat(Features), + desc("Experimental support for standard c++ modules"), + init(false), +}; + /// Supports a test URI scheme with relaxed constraints for lit tests. /// The path in a test URI will be combined with a platform-specific fake /// directory to form an absolute path. For example, test:///a.cpp is resolved @@ -860,6 +867,7 @@ clangd accepts flags on the commandline, and in the CLANGD_FLAGS environment var ClangdLSPServer::Options Opts; Opts.UseDirBasedCDB = (CompileArgsFrom == FilesystemCompileArgs); + Opts.EnableExperimentalModulesSupport = ExperimentalModulesSupport; switch (PCHStorage) { case PCHStorageFlag::Memory: diff --git a/clang-tools-extra/clangd/unittests/CMakeLists.txt b/clang-tools-extra/clangd/unittests/CMakeLists.txt index 0d4628ccf25d8c9..4fa9f18407ae9e6 100644 --- a/clang-tools-extra/clangd/unittests/CMakeLists.txt +++ b/clang-tools-extra/clangd/unittests/CMakeLists.txt @@ -74,6 +74,7 @@ add_unittest(ClangdUnitTests ClangdTests LoggerTests.cpp LSPBinderTests.cpp LSPClient.cpp + PrerequisiteModulesTest.cpp ModulesTests.cpp ParsedASTTests.cpp PathMappingTests.cpp diff --git a/clang-tools-extra/clangd/unittests/PrerequisiteModulesTest.cpp b/clang-tools-extra/clangd/unittests/PrerequisiteModulesTest.cpp new file mode 100644 index 000000000000000..7bbb95c8b8d67f4 --- /dev/null +++ b/clang-tools-extra/clangd/unittests/PrerequisiteModulesTest.cpp @@ -0,0 +1,408 @@ +//===--------------- PrerequisiteModulesTests.cpp -------------------*- C++ +//-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +/// FIXME: Skip testing on windows temporarily due to the different escaping +/// code mode. +#ifndef _WIN32 + +#include "ModulesBuilder.h" +#include "ScanningProjectModules.h" +#include "Annotations.h" +#include "CodeComplete.h" +#include "Compiler.h" +#include "TestTU.h" +#include "support/ThreadsafeFS.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/raw_ostream.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace clang::clangd { +namespace { + +class MockDirectoryCompilationDatabase : public MockCompilationDatabase { +public: + MockDirectoryCompilationDatabase(StringRef TestDir, const ThreadsafeFS &TFS) + : MockCompilationDatabase(TestDir), + MockedCDBPtr(std::make_shared(*this)), + TFS(TFS) { + this->ExtraClangFlags.push_back("-std=c++20"); + this->ExtraClangFlags.push_back("-c"); + } + + void addFile(llvm::StringRef Path, llvm::StringRef Contents); + + std::unique_ptr getProjectModules(PathRef) const override { + return scanningProjectModules(MockedCDBPtr, TFS); + } + +private: + class MockClangCompilationDatabase : public tooling::CompilationDatabase { + public: + MockClangCompilationDatabase(MockDirectoryCompilationDatabase &MCDB) + : MCDB(MCDB) {} + + std::vector + getCompileCommands(StringRef FilePath) const override { + std::optional Cmd = + MCDB.getCompileCommand(FilePath); + EXPECT_TRUE(Cmd); + return {*Cmd}; + } + + std::vector getAllFiles() const override { return Files; } + + void AddFile(StringRef File) { Files.push_back(File.str()); } + + private: + MockDirectoryCompilationDatabase &MCDB; + std::vector Files; + }; + + std::shared_ptr MockedCDBPtr; + const ThreadsafeFS &TFS; +}; + +// Add files to the working testing directory and the compilation database. +void MockDirectoryCompilationDatabase::addFile(llvm::StringRef Path, + llvm::StringRef Contents) { + ASSERT_FALSE(llvm::sys::path::is_absolute(Path)); + + SmallString<256> AbsPath(Directory); + llvm::sys::path::append(AbsPath, Path); + + ASSERT_FALSE( + llvm::sys::fs::create_directories(llvm::sys::path::parent_path(AbsPath))); + + std::error_code EC; + llvm::raw_fd_ostream OS(AbsPath, EC); + ASSERT_FALSE(EC); + OS << Contents; + + MockedCDBPtr->AddFile(Path); +} + +class PrerequisiteModulesTests : public ::testing::Test { +protected: + void SetUp() override { + ASSERT_FALSE(llvm::sys::fs::createUniqueDirectory("modules-test", TestDir)); + } + + void TearDown() override { + ASSERT_FALSE(llvm::sys::fs::remove_directories(TestDir)); + } + +public: + // Get the absolute path for file specified by Path under testing working + // directory. + std::string getFullPath(llvm::StringRef Path) { + SmallString<128> Result(TestDir); + llvm::sys::path::append(Result, Path); + EXPECT_TRUE(llvm::sys::fs::exists(Result.str())); + return Result.str().str(); + } + + ParseInputs getInputs(llvm::StringRef FileName, + const GlobalCompilationDatabase &CDB) { + std::string FullPathName = getFullPath(FileName); + + ParseInputs Inputs; + std::optional Cmd = + CDB.getCompileCommand(FullPathName); + EXPECT_TRUE(Cmd); + Inputs.CompileCommand = std::move(*Cmd); + Inputs.TFS = &FS; + + if (auto Contents = FS.view(TestDir)->getBufferForFile(FullPathName)) + Inputs.Contents = Contents->get()->getBuffer().str(); + + return Inputs; + } + + SmallString<256> TestDir; + // FIXME: It will be better to use the MockFS if the scanning process and + // build module process doesn't depend on reading real IO. + RealThreadsafeFS FS; + + DiagnosticConsumer DiagConsumer; +}; + +TEST_F(PrerequisiteModulesTests, NonModularTest) { + MockDirectoryCompilationDatabase CDB(TestDir, FS); + + CDB.addFile("foo.h", R"cpp( +inline void foo() {} + )cpp"); + + CDB.addFile("NonModular.cpp", R"cpp( +#include "foo.h" +void use() { + foo(); +} + )cpp"); + + ModulesBuilder Builder(CDB); + + // NonModular.cpp is not related to modules. So nothing should be built. + auto NonModularInfo = + Builder.buildPrerequisiteModulesFor(getFullPath("NonModular.cpp"), FS); + EXPECT_TRUE(NonModularInfo); + + HeaderSearchOptions HSOpts; + NonModularInfo->adjustHeaderSearchOptions(HSOpts); + EXPECT_TRUE(HSOpts.PrebuiltModuleFiles.empty()); + + auto Invocation = + buildCompilerInvocation(getInputs("NonModular.cpp", CDB), DiagConsumer); + EXPECT_TRUE(NonModularInfo->canReuse(*Invocation, FS.view(TestDir))); +} + +TEST_F(PrerequisiteModulesTests, ModuleWithoutDepTest) { + MockDirectoryCompilationDatabase CDB(TestDir, FS); + + CDB.addFile("foo.h", R"cpp( +inline void foo() {} + )cpp"); + + CDB.addFile("M.cppm", R"cpp( +module; +#include "foo.h" +export module M; + )cpp"); + + ModulesBuilder Builder(CDB); + + auto MInfo = Builder.buildPrerequisiteModulesFor(getFullPath("M.cppm"), FS); + EXPECT_TRUE(MInfo); + + // Nothing should be built since M doesn't dependent on anything. + HeaderSearchOptions HSOpts; + MInfo->adjustHeaderSearchOptions(HSOpts); + EXPECT_TRUE(HSOpts.PrebuiltModuleFiles.empty()); + + auto Invocation = + buildCompilerInvocation(getInputs("M.cppm", CDB), DiagConsumer); + EXPECT_TRUE(MInfo->canReuse(*Invocation, FS.view(TestDir))); +} + +TEST_F(PrerequisiteModulesTests, ModuleWithDepTest) { + MockDirectoryCompilationDatabase CDB(TestDir, FS); + + CDB.addFile("foo.h", R"cpp( +inline void foo() {} + )cpp"); + + CDB.addFile("M.cppm", R"cpp( +module; +#include "foo.h" +export module M; + )cpp"); + + CDB.addFile("N.cppm", R"cpp( +export module N; +import :Part; +import M; + )cpp"); + + CDB.addFile("N-part.cppm", R"cpp( +// Different module name with filename intentionally. +export module N:Part; + )cpp"); + + ModulesBuilder Builder(CDB); + + auto NInfo = Builder.buildPrerequisiteModulesFor(getFullPath("N.cppm"), FS); + EXPECT_TRUE(NInfo); + + ParseInputs NInput = getInputs("N.cppm", CDB); + std::unique_ptr Invocation = + buildCompilerInvocation(NInput, DiagConsumer); + // Test that `PrerequisiteModules::canReuse` works basically. + EXPECT_TRUE(NInfo->canReuse(*Invocation, FS.view(TestDir))); + + { + // Check that + // `PrerequisiteModules::adjustHeaderSearchOptions(HeaderSearchOptions&)` + // can appending HeaderSearchOptions correctly. + HeaderSearchOptions HSOpts; + NInfo->adjustHeaderSearchOptions(HSOpts); + + EXPECT_TRUE(HSOpts.PrebuiltModuleFiles.count("M")); + EXPECT_TRUE(HSOpts.PrebuiltModuleFiles.count("N:Part")); + } + + { + // Check that + // `PrerequisiteModules::adjustHeaderSearchOptions(HeaderSearchOptions&)` + // can replace HeaderSearchOptions correctly. + HeaderSearchOptions HSOpts; + HSOpts.PrebuiltModuleFiles["M"] = "incorrect_path"; + HSOpts.PrebuiltModuleFiles["N:Part"] = "incorrect_path"; + NInfo->adjustHeaderSearchOptions(HSOpts); + + EXPECT_TRUE(StringRef(HSOpts.PrebuiltModuleFiles["M"]).ends_with(".pcm")); + EXPECT_TRUE( + StringRef(HSOpts.PrebuiltModuleFiles["N:Part"]).ends_with(".pcm")); + } +} + +TEST_F(PrerequisiteModulesTests, ReusabilityTest) { + MockDirectoryCompilationDatabase CDB(TestDir, FS); + + CDB.addFile("foo.h", R"cpp( +inline void foo() {} + )cpp"); + + CDB.addFile("M.cppm", R"cpp( +module; +#include "foo.h" +export module M; + )cpp"); + + CDB.addFile("N.cppm", R"cpp( +export module N; +import :Part; +import M; + )cpp"); + + CDB.addFile("N-part.cppm", R"cpp( +// Different module name with filename intentionally. +export module N:Part; + )cpp"); + + ModulesBuilder Builder(CDB); + + auto NInfo = Builder.buildPrerequisiteModulesFor(getFullPath("N.cppm"), FS); + EXPECT_TRUE(NInfo); + EXPECT_TRUE(NInfo); + + ParseInputs NInput = getInputs("N.cppm", CDB); + std::unique_ptr Invocation = + buildCompilerInvocation(NInput, DiagConsumer); + EXPECT_TRUE(NInfo->canReuse(*Invocation, FS.view(TestDir))); + + // Test that we can still reuse the NInfo after we touch a unrelated file. + { + CDB.addFile("L.cppm", R"cpp( +module; +#include "foo.h" +export module L; +export int ll = 43; + )cpp"); + EXPECT_TRUE(NInfo->canReuse(*Invocation, FS.view(TestDir))); + + CDB.addFile("bar.h", R"cpp( +inline void bar() {} +inline void bar(int) {} + )cpp"); + EXPECT_TRUE(NInfo->canReuse(*Invocation, FS.view(TestDir))); + } + + // Test that we can't reuse the NInfo after we touch a related file. + { + CDB.addFile("M.cppm", R"cpp( +module; +#include "foo.h" +export module M; +export int mm = 44; + )cpp"); + EXPECT_FALSE(NInfo->canReuse(*Invocation, FS.view(TestDir))); + + NInfo = Builder.buildPrerequisiteModulesFor(getFullPath("N.cppm"), FS); + EXPECT_TRUE(NInfo->canReuse(*Invocation, FS.view(TestDir))); + + CDB.addFile("foo.h", R"cpp( +inline void foo() {} +inline void foo(int) {} + )cpp"); + EXPECT_FALSE(NInfo->canReuse(*Invocation, FS.view(TestDir))); + + NInfo = Builder.buildPrerequisiteModulesFor(getFullPath("N.cppm"), FS); + EXPECT_TRUE(NInfo->canReuse(*Invocation, FS.view(TestDir))); + } + + CDB.addFile("N-part.cppm", R"cpp( +export module N:Part; +// Intentioned to make it uncompilable. +export int NPart = 4LIdjwldijaw + )cpp"); + EXPECT_FALSE(NInfo->canReuse(*Invocation, FS.view(TestDir))); + NInfo = Builder.buildPrerequisiteModulesFor(getFullPath("N.cppm"), FS); + EXPECT_TRUE(NInfo); + EXPECT_FALSE(NInfo->canReuse(*Invocation, FS.view(TestDir))); + + CDB.addFile("N-part.cppm", R"cpp( +export module N:Part; +export int NPart = 43; + )cpp"); + EXPECT_TRUE(NInfo); + EXPECT_FALSE(NInfo->canReuse(*Invocation, FS.view(TestDir))); + NInfo = Builder.buildPrerequisiteModulesFor(getFullPath("N.cppm"), FS); + EXPECT_TRUE(NInfo); + EXPECT_TRUE(NInfo->canReuse(*Invocation, FS.view(TestDir))); + + // Test that if we changed the modification time of the file, the module files + // info is still reusable if its content doesn't change. + CDB.addFile("N-part.cppm", R"cpp( +export module N:Part; +export int NPart = 43; + )cpp"); + EXPECT_TRUE(NInfo->canReuse(*Invocation, FS.view(TestDir))); + + CDB.addFile("N.cppm", R"cpp( +export module N; +import :Part; +import M; + +export int nn = 43; + )cpp"); + // NInfo should be reusable after we change its content. + EXPECT_TRUE(NInfo->canReuse(*Invocation, FS.view(TestDir))); +} + +// An End-to-End test for modules. +TEST_F(PrerequisiteModulesTests, ParsedASTTest) { + MockDirectoryCompilationDatabase CDB(TestDir, FS); + + CDB.addFile("A.cppm", R"cpp( +export module A; +export void printA(); + )cpp"); + + CDB.addFile("Use.cpp", R"cpp( +import A; +)cpp"); + + ModulesBuilder Builder(CDB); + + ParseInputs Use = getInputs("Use.cpp", CDB); + Use.ModulesManager = &Builder; + + std::unique_ptr CI = + buildCompilerInvocation(Use, DiagConsumer); + EXPECT_TRUE(CI); + + auto Preamble = + buildPreamble(getFullPath("Use.cpp"), *CI, Use, /*InMemory=*/true, + /*Callback=*/nullptr); + EXPECT_TRUE(Preamble); + EXPECT_TRUE(Preamble->RequiredModules); + + auto AST = ParsedAST::build(getFullPath("Use.cpp"), Use, std::move(CI), {}, + Preamble); + EXPECT_TRUE(AST); + + const NamedDecl &D = findDecl(*AST, "printA"); + EXPECT_TRUE(D.isFromASTFile()); +} + +} // namespace +} // namespace clang::clangd + +#endif diff --git a/clang-tools-extra/clangd/unittests/TestFS.h b/clang-tools-extra/clangd/unittests/TestFS.h index 6bdadc9c07439d8..568533f3b3b9114 100644 --- a/clang-tools-extra/clangd/unittests/TestFS.h +++ b/clang-tools-extra/clangd/unittests/TestFS.h @@ -67,7 +67,7 @@ class MockCompilationDatabase : public GlobalCompilationDatabase { std::vector ExtraClangFlags; -private: +protected: StringRef Directory; StringRef RelPathPrefix; }; diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst index 004811d2eca4f49..697b514ae1572ac 100644 --- a/clang-tools-extra/docs/ReleaseNotes.rst +++ b/clang-tools-extra/docs/ReleaseNotes.rst @@ -48,6 +48,10 @@ Major New Features Improvements to clangd ---------------------- +- Introduced exmperimental support for C++20 Modules. The experimental support can + be enabled by `-experimental-modules-support` option. It is in an early development + stage and may not perform efficiently in real-world scenarios. + Inlay hints ^^^^^^^^^^^