diff --git a/clang-tools-extra/clangd/CMakeLists.txt b/clang-tools-extra/clangd/CMakeLists.txt index f49704157880d3a..c21d277d2ffcbd8 100644 --- a/clang-tools-extra/clangd/CMakeLists.txt +++ b/clang-tools-extra/clangd/CMakeLists.txt @@ -97,12 +97,14 @@ add_clang_library(clangDaemon IncludeFixer.cpp InlayHints.cpp JSONTransport.cpp + ModulesBuilder.cpp PathMapping.cpp Protocol.cpp Quality.cpp ParsedAST.cpp Preamble.cpp RIFF.cpp + ScanningProjectModules.cpp Selection.cpp SemanticHighlighting.cpp SemanticSelection.cpp @@ -161,6 +163,7 @@ clang_target_link_libraries(clangDaemon clangAST clangASTMatchers clangBasic + clangDependencyScanning clangDriver clangFormat clangFrontend diff --git a/clang-tools-extra/clangd/ClangdLSPServer.cpp b/clang-tools-extra/clangd/ClangdLSPServer.cpp index 7fd599d4e1a0b0e..06573a575542451 100644 --- a/clang-tools-extra/clangd/ClangdLSPServer.cpp +++ b/clang-tools-extra/clangd/ClangdLSPServer.cpp @@ -14,6 +14,7 @@ #include "Feature.h" #include "GlobalCompilationDatabase.h" #include "LSPBinder.h" +#include "ModulesBuilder.h" #include "Protocol.h" #include "SemanticHighlighting.h" #include "SourceCode.h" @@ -51,6 +52,7 @@ namespace clang { namespace clangd { + namespace { // Tracks end-to-end latency of high level lsp calls. Measurements are in // seconds. @@ -563,6 +565,12 @@ void ClangdLSPServer::onInitialize(const InitializeParams &Params, Mangler.ResourceDir = *Opts.ResourceDir; CDB.emplace(BaseCDB.get(), Params.initializationOptions.fallbackFlags, std::move(Mangler)); + + if (Opts.EnableExperimentalModulesSupport) { + ModulesManager.emplace(*CDB); + Opts.ModulesManager = &*ModulesManager; + } + { // Switch caller's context with LSPServer's background context. Since we // rather want to propagate information from LSPServer's context into the diff --git a/clang-tools-extra/clangd/ClangdLSPServer.h b/clang-tools-extra/clangd/ClangdLSPServer.h index 8bcb29522509b75..0b8e4720f53236b 100644 --- a/clang-tools-extra/clangd/ClangdLSPServer.h +++ b/clang-tools-extra/clangd/ClangdLSPServer.h @@ -63,6 +63,9 @@ class ClangdLSPServer : private ClangdServer::Callbacks, /// Limit the number of references returned (0 means no limit). size_t ReferencesLimit = 0; + + /// Flag to hint the experimental modules support is enabled. + bool EnableExperimentalModulesSupport = false; }; ClangdLSPServer(Transport &Transp, const ThreadsafeFS &TFS, @@ -323,6 +326,8 @@ class ClangdLSPServer : private ClangdServer::Callbacks, std::optional CDB; // The ClangdServer is created by the "initialize" LSP method. std::optional Server; + // Manages to build module files. + std::optional ModulesManager; }; } // namespace clangd } // namespace clang diff --git a/clang-tools-extra/clangd/ClangdServer.cpp b/clang-tools-extra/clangd/ClangdServer.cpp index 1c4c2a79b5c0510..e910a80ba0bae9b 100644 --- a/clang-tools-extra/clangd/ClangdServer.cpp +++ b/clang-tools-extra/clangd/ClangdServer.cpp @@ -216,6 +216,7 @@ ClangdServer::ClangdServer(const GlobalCompilationDatabase &CDB, Callbacks *Callbacks) : FeatureModules(Opts.FeatureModules), CDB(CDB), TFS(TFS), DynamicIdx(Opts.BuildDynamicSymbolIndex ? new FileIndex() : nullptr), + ModulesManager(Opts.ModulesManager), ClangTidyProvider(Opts.ClangTidyProvider), UseDirtyHeaders(Opts.UseDirtyHeaders), LineFoldingOnly(Opts.LineFoldingOnly), @@ -308,6 +309,7 @@ void ClangdServer::addDocument(PathRef File, llvm::StringRef Contents, Inputs.Index = Index; Inputs.ClangTidyProvider = ClangTidyProvider; Inputs.FeatureModules = FeatureModules; + Inputs.ModulesManager = ModulesManager; bool NewFile = WorkScheduler->update(File, Inputs, WantDiags); // If we loaded Foo.h, we want to make sure Foo.cpp is indexed. if (NewFile && BackgroundIdx) diff --git a/clang-tools-extra/clangd/ClangdServer.h b/clang-tools-extra/clangd/ClangdServer.h index 1661028be88b4e7..a653cdb56b751b6 100644 --- a/clang-tools-extra/clangd/ClangdServer.h +++ b/clang-tools-extra/clangd/ClangdServer.h @@ -16,6 +16,7 @@ #include "FeatureModule.h" #include "GlobalCompilationDatabase.h" #include "Hover.h" +#include "ModulesBuilder.h" #include "Protocol.h" #include "SemanticHighlighting.h" #include "TUScheduler.h" @@ -112,6 +113,9 @@ class ClangdServer { /// This throttler controls which preambles may be built at a given time. clangd::PreambleThrottler *PreambleThrottler = nullptr; + /// Manages to build module files. + ModulesBuilder *ModulesManager = nullptr; + /// If true, ClangdServer builds a dynamic in-memory index for symbols in /// opened files and uses the index to augment code completion results. bool BuildDynamicSymbolIndex = false; @@ -477,6 +481,8 @@ class ClangdServer { std::unique_ptr BackgroundIdx; // Storage for merged views of the various indexes. std::vector> MergedIdx; + // Manage module files. + ModulesBuilder *ModulesManager = nullptr; // When set, provides clang-tidy options for a specific file. TidyProviderRef ClangTidyProvider; diff --git a/clang-tools-extra/clangd/Compiler.h b/clang-tools-extra/clangd/Compiler.h index 56c2567ebe97b0c..4e68da7610ca2c3 100644 --- a/clang-tools-extra/clangd/Compiler.h +++ b/clang-tools-extra/clangd/Compiler.h @@ -16,6 +16,7 @@ #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_COMPILER_H #include "FeatureModule.h" +#include "ModulesBuilder.h" #include "TidyProvider.h" #include "index/Index.h" #include "support/ThreadsafeFS.h" @@ -60,6 +61,8 @@ struct ParseInputs { TidyProviderRef ClangTidyProvider = {}; // Used to acquire ASTListeners when parsing files. FeatureModuleSet *FeatureModules = nullptr; + // Used to build and manage (C++) modules. + ModulesBuilder *ModulesManager = nullptr; }; /// Clears \p CI from options that are not supported by clangd, like codegen or diff --git a/clang-tools-extra/clangd/GlobalCompilationDatabase.cpp b/clang-tools-extra/clangd/GlobalCompilationDatabase.cpp index 85c80eb482efbf3..1d96667a8e9f4a7 100644 --- a/clang-tools-extra/clangd/GlobalCompilationDatabase.cpp +++ b/clang-tools-extra/clangd/GlobalCompilationDatabase.cpp @@ -9,6 +9,8 @@ #include "GlobalCompilationDatabase.h" #include "Config.h" #include "FS.h" +#include "ProjectModules.h" +#include "ScanningProjectModules.h" #include "SourceCode.h" #include "support/Logger.h" #include "support/Path.h" @@ -741,6 +743,20 @@ DirectoryBasedGlobalCompilationDatabase::getProjectInfo(PathRef File) const { return Res->PI; } +std::unique_ptr +DirectoryBasedGlobalCompilationDatabase::getProjectModules(PathRef File) const { + CDBLookupRequest Req; + Req.FileName = File; + Req.ShouldBroadcast = false; + Req.FreshTime = Req.FreshTimeMissing = + std::chrono::steady_clock::time_point::min(); + auto Res = lookupCDB(Req); + if (!Res) + return {}; + + return scanningProjectModules(Res->CDB, Opts.TFS); +} + OverlayCDB::OverlayCDB(const GlobalCompilationDatabase *Base, std::vector FallbackFlags, CommandMangler Mangler) @@ -833,6 +849,13 @@ std::optional DelegatingCDB::getProjectInfo(PathRef File) const { return Base->getProjectInfo(File); } +std::unique_ptr +DelegatingCDB::getProjectModules(PathRef File) const { + if (!Base) + return nullptr; + return Base->getProjectModules(File); +} + tooling::CompileCommand DelegatingCDB::getFallbackCommand(PathRef File) const { if (!Base) return GlobalCompilationDatabase::getFallbackCommand(File); diff --git a/clang-tools-extra/clangd/GlobalCompilationDatabase.h b/clang-tools-extra/clangd/GlobalCompilationDatabase.h index 2bf8c973c534c6f..ea999fe8aee0177 100644 --- a/clang-tools-extra/clangd/GlobalCompilationDatabase.h +++ b/clang-tools-extra/clangd/GlobalCompilationDatabase.h @@ -9,6 +9,7 @@ #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_GLOBALCOMPILATIONDATABASE_H #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_GLOBALCOMPILATIONDATABASE_H +#include "ProjectModules.h" #include "support/Function.h" #include "support/Path.h" #include "support/Threading.h" @@ -45,6 +46,12 @@ class GlobalCompilationDatabase { return std::nullopt; } + /// Get the modules in the closest project to \p File + virtual std::unique_ptr + getProjectModules(PathRef File) const { + return nullptr; + } + /// Makes a guess at how to build a file. /// The default implementation just runs clang on the file. /// Clangd should treat the results as unreliable. @@ -76,6 +83,9 @@ class DelegatingCDB : public GlobalCompilationDatabase { std::optional getProjectInfo(PathRef File) const override; + std::unique_ptr + getProjectModules(PathRef File) const override; + tooling::CompileCommand getFallbackCommand(PathRef File) const override; bool blockUntilIdle(Deadline D) const override; @@ -122,6 +132,9 @@ class DirectoryBasedGlobalCompilationDatabase /// \p File's parents. std::optional getProjectInfo(PathRef File) const override; + std::unique_ptr + getProjectModules(PathRef File) const override; + bool blockUntilIdle(Deadline Timeout) const override; private: diff --git a/clang-tools-extra/clangd/ModulesBuilder.cpp b/clang-tools-extra/clangd/ModulesBuilder.cpp new file mode 100644 index 000000000000000..94c7eec2d09e4ee --- /dev/null +++ b/clang-tools-extra/clangd/ModulesBuilder.cpp @@ -0,0 +1,336 @@ +//===----------------- ModulesBuilder.cpp ------------------------*- C++-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "ModulesBuilder.h" +#include "Compiler.h" +#include "support/Logger.h" +#include "clang/Frontend/FrontendAction.h" +#include "clang/Frontend/FrontendActions.h" +#include "clang/Serialization/ASTReader.h" + +namespace clang { +namespace clangd { + +namespace { + +// Create a path to store module files. Generally it should be: +// +// {TEMP_DIRS}/clangd/module_files/{hashed-file-name}-%%-%%-%%-%%-%%-%%/. +// +// {TEMP_DIRS} is the temporary directory for the system, e.g., "/var/tmp" +// or "C:/TEMP". +// +// '%%' means random value to make the generated path unique. +// +// \param MainFile is used to get the root of the project from global +// compilation database. +// +// TODO: Move these module fils out of the temporary directory if the module +// files are persistent. +llvm::SmallString<256> getUniqueModuleFilesPath(PathRef MainFile) { + llvm::SmallString<128> HashedPrefix = llvm::sys::path::filename(MainFile); + // There might be multiple files with the same name in a project. So appending + // the hash value of the full path to make sure they won't conflict. + HashedPrefix += std::to_string(llvm::hash_value(MainFile)); + + llvm::SmallString<256> ResultPattern; + + llvm::sys::path::system_temp_directory(/*erasedOnReboot=*/true, + ResultPattern); + + llvm::sys::path::append(ResultPattern, "clangd"); + llvm::sys::path::append(ResultPattern, "module_files"); + + llvm::sys::path::append(ResultPattern, HashedPrefix); + + ResultPattern.append("-%%-%%-%%-%%-%%-%%"); + + llvm::SmallString<256> Result; + llvm::sys::fs::createUniquePath(ResultPattern, Result, + /*MakeAbsolute=*/false); + + llvm::sys::fs::create_directories(Result); + return Result; +} + +// Get a unique module file path under \param ModuleFilesPrefix. +std::string getModuleFilePath(llvm::StringRef ModuleName, + PathRef ModuleFilesPrefix) { + llvm::SmallString<256> ModuleFilePath(ModuleFilesPrefix); + auto [PrimaryModuleName, PartitionName] = ModuleName.split(':'); + llvm::sys::path::append(ModuleFilePath, PrimaryModuleName); + if (!PartitionName.empty()) { + ModuleFilePath.append("-"); + ModuleFilePath.append(PartitionName); + } + + ModuleFilePath.append(".pcm"); + return std::string(ModuleFilePath); +} + +// FailedPrerequisiteModules - stands for the PrerequisiteModules which has +// errors happened during the building process. +class FailedPrerequisiteModules : public PrerequisiteModules { +public: + ~FailedPrerequisiteModules() override = default; + + // We shouldn't adjust the compilation commands based on + // FailedPrerequisiteModules. + void adjustHeaderSearchOptions(HeaderSearchOptions &Options) const override { + } + + // FailedPrerequisiteModules can never be reused. + bool + canReuse(const CompilerInvocation &CI, + llvm::IntrusiveRefCntPtr) const override { + return false; + } +}; + +// StandalonePrerequisiteModules - stands for PrerequisiteModules for which all +// the required modules are built successfully. All the module files +// are owned by the StandalonePrerequisiteModules class. +// +// Any of the built module files won't be shared with other instances of the +// class. So that we can avoid worrying thread safety. +// +// We don't need to worry about duplicated module names here since the standard +// guarantees the module names should be unique to a program. +class StandalonePrerequisiteModules : public PrerequisiteModules { +public: + StandalonePrerequisiteModules() = default; + + StandalonePrerequisiteModules(const StandalonePrerequisiteModules &) = delete; + StandalonePrerequisiteModules + operator=(const StandalonePrerequisiteModules &) = delete; + StandalonePrerequisiteModules(StandalonePrerequisiteModules &&) = delete; + StandalonePrerequisiteModules + operator=(StandalonePrerequisiteModules &&) = delete; + + ~StandalonePrerequisiteModules() override = default; + + void adjustHeaderSearchOptions(HeaderSearchOptions &Options) const override { + // Appending all built module files. + for (auto &RequiredModule : RequiredModules) + Options.PrebuiltModuleFiles.insert_or_assign( + RequiredModule.ModuleName, RequiredModule.ModuleFilePath); + } + + bool canReuse(const CompilerInvocation &CI, + llvm::IntrusiveRefCntPtr) const override; + + bool isModuleUnitBuilt(llvm::StringRef ModuleName) const { + return BuiltModuleNames.contains(ModuleName); + } + + void addModuleFile(llvm::StringRef ModuleName, + llvm::StringRef ModuleFilePath) { + RequiredModules.emplace_back(ModuleName, ModuleFilePath); + BuiltModuleNames.insert(ModuleName); + } + +private: + struct ModuleFile { + ModuleFile(llvm::StringRef ModuleName, PathRef ModuleFilePath) + : ModuleName(ModuleName.str()), ModuleFilePath(ModuleFilePath.str()) {} + + ModuleFile(const ModuleFile &) = delete; + ModuleFile operator=(const ModuleFile &) = delete; + + // The move constructor is needed for llvm::SmallVector. + ModuleFile(ModuleFile &&Other) + : ModuleName(std::move(Other.ModuleName)), + ModuleFilePath(std::move(Other.ModuleFilePath)) {} + + ModuleFile &operator=(ModuleFile &&Other) = delete; + + ~ModuleFile() { + if (!ModuleFilePath.empty()) + llvm::sys::fs::remove(ModuleFilePath); + } + + std::string ModuleName; + std::string ModuleFilePath; + }; + + llvm::SmallVector RequiredModules; + // A helper class to speedup the query if a module is built. + llvm::StringSet<> BuiltModuleNames; +}; + +// Build a module file for module with `ModuleName`. The information of built +// module file are stored in \param BuiltModuleFiles. +llvm::Error buildModuleFile(llvm::StringRef ModuleName, + const GlobalCompilationDatabase &CDB, + const ThreadsafeFS &TFS, ProjectModules &MDB, + PathRef ModuleFilesPrefix, + StandalonePrerequisiteModules &BuiltModuleFiles) { + if (BuiltModuleFiles.isModuleUnitBuilt(ModuleName)) + return llvm::Error::success(); + + PathRef ModuleUnitFileName = MDB.getSourceForModuleName(ModuleName); + // It is possible that we're meeting third party modules (modules whose + // source are not in the project. e.g, the std module may be a third-party + // module for most projects) or something wrong with the implementation of + // ProjectModules. + // FIXME: How should we treat third party modules here? If we want to ignore + // third party modules, we should return true instead of false here. + // Currently we simply bail out. + if (ModuleUnitFileName.empty()) + return llvm::createStringError("Failed to get the primary source"); + + // Try cheap operation earlier to boil-out cheaply if there are problems. + auto Cmd = CDB.getCompileCommand(ModuleUnitFileName); + if (!Cmd) + return llvm::createStringError( + llvm::formatv("No compile command for {0}", ModuleUnitFileName)); + + for (auto &RequiredModuleName : MDB.getRequiredModules(ModuleUnitFileName)) { + // Return early if there are errors building the module file. + if (llvm::Error Err = buildModuleFile(RequiredModuleName, CDB, TFS, MDB, + ModuleFilesPrefix, BuiltModuleFiles)) + return llvm::createStringError( + llvm::formatv("Failed to build dependency {0}: {1}", + RequiredModuleName, llvm::toString(std::move(Err)))); + } + + Cmd->Output = getModuleFilePath(ModuleName, ModuleFilesPrefix); + + ParseInputs Inputs; + Inputs.TFS = &TFS; + Inputs.CompileCommand = std::move(*Cmd); + + IgnoreDiagnostics IgnoreDiags; + auto CI = buildCompilerInvocation(Inputs, IgnoreDiags); + if (!CI) + return llvm::createStringError("Failed to build compiler invocation"); + + auto FS = Inputs.TFS->view(Inputs.CompileCommand.Directory); + auto Buf = FS->getBufferForFile(Inputs.CompileCommand.Filename); + if (!Buf) + return llvm::createStringError("Failed to create buffer"); + + // In clang's driver, we will suppress the check for ODR violation in GMF. + // See the implementation of RenderModulesOptions in Clang.cpp. + CI->getLangOpts().SkipODRCheckInGMF = true; + + // Hash the contents of input files and store the hash value to the BMI files. + // So that we can check if the files are still valid when we want to reuse the + // BMI files. + CI->getHeaderSearchOpts().ValidateASTInputFilesContent = true; + + BuiltModuleFiles.adjustHeaderSearchOptions(CI->getHeaderSearchOpts()); + + CI->getFrontendOpts().OutputFile = Inputs.CompileCommand.Output; + auto Clang = + prepareCompilerInstance(std::move(CI), /*Preamble=*/nullptr, + std::move(*Buf), std::move(FS), IgnoreDiags); + if (!Clang) + return llvm::createStringError("Failed to prepare compiler instance"); + + GenerateReducedModuleInterfaceAction Action; + Clang->ExecuteAction(Action); + + if (Clang->getDiagnostics().hasErrorOccurred()) + return llvm::createStringError("Compilation failed"); + + BuiltModuleFiles.addModuleFile(ModuleName, Inputs.CompileCommand.Output); + return llvm::Error::success(); +} +} // namespace + +std::unique_ptr +ModulesBuilder::buildPrerequisiteModulesFor(PathRef File, + const ThreadsafeFS &TFS) const { + std::unique_ptr MDB = CDB.getProjectModules(File); + if (!MDB) { + elog("Failed to get Project Modules information for {0}", File); + return std::make_unique(); + } + + std::vector RequiredModuleNames = MDB->getRequiredModules(File); + if (RequiredModuleNames.empty()) + return std::make_unique(); + + llvm::SmallString<256> ModuleFilesPrefix = getUniqueModuleFilesPath(File); + + log("Trying to build required modules for {0} in {1}", File, + ModuleFilesPrefix); + + auto RequiredModules = std::make_unique(); + + for (llvm::StringRef RequiredModuleName : RequiredModuleNames) { + // Return early if there is any error. + if (llvm::Error Err = + buildModuleFile(RequiredModuleName, CDB, TFS, *MDB.get(), + ModuleFilesPrefix, *RequiredModules.get())) { + elog("Failed to build module {0}; due to {1}", RequiredModuleName, + toString(std::move(Err))); + return std::make_unique(); + } + } + + log("Built required modules for {0} in {1}", File, ModuleFilesPrefix); + + return std::move(RequiredModules); +} + +bool StandalonePrerequisiteModules::canReuse( + const CompilerInvocation &CI, + llvm::IntrusiveRefCntPtr VFS) const { + if (RequiredModules.empty()) + return true; + + CompilerInstance Clang; + + Clang.setInvocation(std::make_shared(CI)); + IntrusiveRefCntPtr Diags = + CompilerInstance::createDiagnostics(new DiagnosticOptions()); + Clang.setDiagnostics(Diags.get()); + + FileManager *FM = Clang.createFileManager(VFS); + Clang.createSourceManager(*FM); + + if (!Clang.createTarget()) + return false; + + assert(Clang.getHeaderSearchOptsPtr()); + adjustHeaderSearchOptions(Clang.getHeaderSearchOpts()); + // Since we don't need to compile the source code actually, the TU kind here + // doesn't matter. + Clang.createPreprocessor(TU_Complete); + Clang.getHeaderSearchOpts().ForceCheckCXX20ModulesInputFiles = true; + Clang.getHeaderSearchOpts().ValidateASTInputFilesContent = true; + + // Following the practice of clang's driver to suppres the checking for ODR + // violation in GMF. + // See + // https://clang.llvm.org/docs/StandardCPlusPlusModules.html#object-definition-consistency + // for example. + Clang.getLangOpts().SkipODRCheckInGMF = true; + + Clang.createASTReader(); + for (auto &RequiredModule : RequiredModules) { + llvm::StringRef BMIPath = RequiredModule.ModuleFilePath; + // FIXME: Loading BMI fully is too heavy considering something cheaply to + // check if we can reuse the BMI. + auto ReadResult = + Clang.getASTReader()->ReadAST(BMIPath, serialization::MK_MainFile, + SourceLocation(), ASTReader::ARR_None); + + if (ReadResult != ASTReader::Success) { + elog("Can't reuse {0}: {1}", BMIPath, ReadResult); + return false; + } + } + + return true; +} + +} // namespace clangd +} // namespace clang diff --git a/clang-tools-extra/clangd/ModulesBuilder.h b/clang-tools-extra/clangd/ModulesBuilder.h new file mode 100644 index 000000000000000..0514e7486475d09 --- /dev/null +++ b/clang-tools-extra/clangd/ModulesBuilder.h @@ -0,0 +1,106 @@ +//===----------------- ModulesBuilder.h --------------------------*- C++-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Experimental support for C++20 Modules. +// +// Currently we simplify the implementations by preventing reusing module files +// across different versions and different source files. But this is clearly a +// waste of time and space in the end of the day. +// +// TODO: Supporting reusing module files across different versions and +// different source files. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_MODULES_BUILDER_H +#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_MODULES_BUILDER_H + +#include "GlobalCompilationDatabase.h" +#include "ProjectModules.h" +#include "support/Path.h" +#include "support/ThreadsafeFS.h" +#include "clang/Frontend/CompilerInvocation.h" +#include "llvm/ADT/SmallString.h" +#include + +namespace clang { +namespace clangd { + +/// Store all the needed module files information to parse a single +/// source file. e.g., +/// +/// ``` +/// // a.cppm +/// export module a; +/// +/// // b.cppm +/// export module b; +/// import a; +/// +/// // c.cppm +/// export module c; +/// import b; +/// ``` +/// +/// For the source file `c.cppm`, an instance of the class will store +/// the module files for `a.cppm` and `b.cppm`. But the module file for `c.cppm` +/// won't be stored. Since it is not needed to parse `c.cppm`. +/// +/// Users should only get PrerequisiteModules from +/// `ModulesBuilder::buildPrerequisiteModulesFor(...)`. +/// +/// Users can detect whether the PrerequisiteModules is still up to date by +/// calling the `canReuse()` member function. +/// +/// The users should call `adjustHeaderSearchOptions(...)` to update the +/// compilation commands to select the built module files first. Before calling +/// `adjustHeaderSearchOptions()`, users should call `canReuse()` first to check +/// if all the stored module files are valid. In case they are not valid, +/// users should call `ModulesBuilder::buildPrerequisiteModulesFor(...)` again +/// to get the new PrerequisiteModules. +class PrerequisiteModules { +public: + /// Change commands to load the module files recorded in this + /// PrerequisiteModules first. + virtual void + adjustHeaderSearchOptions(HeaderSearchOptions &Options) const = 0; + + /// Whether or not the built module files are up to date. + /// Note that this can only be used after building the module files. + virtual bool + canReuse(const CompilerInvocation &CI, + llvm::IntrusiveRefCntPtr) const = 0; + + virtual ~PrerequisiteModules() = default; +}; + +/// This class handles building module files for a given source file. +/// +/// In the future, we want the class to manage the module files acorss +/// different versions and different source files. +class ModulesBuilder { +public: + ModulesBuilder(const GlobalCompilationDatabase &CDB) : CDB(CDB) {} + + ModulesBuilder(const ModulesBuilder &) = delete; + ModulesBuilder(ModulesBuilder &&) = delete; + + ModulesBuilder &operator=(const ModulesBuilder &) = delete; + ModulesBuilder &operator=(ModulesBuilder &&) = delete; + + std::unique_ptr + buildPrerequisiteModulesFor(PathRef File, const ThreadsafeFS &TFS) const; + +private: + const GlobalCompilationDatabase &CDB; +}; + +} // namespace clangd +} // namespace clang + +#endif diff --git a/clang-tools-extra/clangd/ParsedAST.cpp b/clang-tools-extra/clangd/ParsedAST.cpp index 2bd1fbcad2ada0f..a2f1504db7e880e 100644 --- a/clang-tools-extra/clangd/ParsedAST.cpp +++ b/clang-tools-extra/clangd/ParsedAST.cpp @@ -446,6 +446,12 @@ ParsedAST::build(llvm::StringRef Filename, const ParseInputs &Inputs, L->sawDiagnostic(D, Diag); }); + // Adjust header search options to load the built module files recorded + // in RequiredModules. + if (Preamble && Preamble->RequiredModules) + Preamble->RequiredModules->adjustHeaderSearchOptions( + CI->getHeaderSearchOpts()); + std::optional Patch; // We might use an ignoring diagnostic consumer if they are going to be // dropped later on to not pay for extra latency by processing them. @@ -459,6 +465,7 @@ ParsedAST::build(llvm::StringRef Filename, const ParseInputs &Inputs, std::move(CI), PreamblePCH, llvm::MemoryBuffer::getMemBufferCopy(Inputs.Contents, Filename), VFS, *DiagConsumer); + if (!Clang) { // The last diagnostic contains information about the reason of this // failure. diff --git a/clang-tools-extra/clangd/Preamble.cpp b/clang-tools-extra/clangd/Preamble.cpp index ecd490145dd3c45..dd13b1a9e5613d2 100644 --- a/clang-tools-extra/clangd/Preamble.cpp +++ b/clang-tools-extra/clangd/Preamble.cpp @@ -664,6 +664,7 @@ buildPreamble(PathRef FileName, CompilerInvocation CI, CI, ContentsBuffer.get(), Bounds, *PreambleDiagsEngine, Stats ? TimedFS : StatCacheFS, std::make_shared(), StoreInMemory, /*StoragePath=*/"", CapturedInfo); + PreambleTimer.stopTimer(); // We have to setup DiagnosticConsumer that will be alife @@ -696,6 +697,19 @@ buildPreamble(PathRef FileName, CompilerInvocation CI, Result->Includes = CapturedInfo.takeIncludes(); Result->Pragmas = std::make_shared( CapturedInfo.takePragmaIncludes()); + + if (Inputs.ModulesManager) { + WallTimer PrerequisiteModuleTimer; + PrerequisiteModuleTimer.startTimer(); + Result->RequiredModules = + Inputs.ModulesManager->buildPrerequisiteModulesFor(FileName, + *Inputs.TFS); + PrerequisiteModuleTimer.stopTimer(); + + log("Built prerequisite modules for file {0} in {1} seconds", FileName, + PrerequisiteModuleTimer.getTime()); + } + Result->Macros = CapturedInfo.takeMacros(); Result->Marks = CapturedInfo.takeMarks(); Result->StatCache = StatCache; @@ -737,7 +751,9 @@ bool isPreambleCompatible(const PreambleData &Preamble, auto VFS = Inputs.TFS->view(Inputs.CompileCommand.Directory); return compileCommandsAreEqual(Inputs.CompileCommand, Preamble.CompileCommand) && - Preamble.Preamble.CanReuse(CI, *ContentsBuffer, Bounds, *VFS); + Preamble.Preamble.CanReuse(CI, *ContentsBuffer, Bounds, *VFS) && + (!Preamble.RequiredModules || + Preamble.RequiredModules->canReuse(CI, VFS)); } void escapeBackslashAndQuotes(llvm::StringRef Text, llvm::raw_ostream &OS) { diff --git a/clang-tools-extra/clangd/Preamble.h b/clang-tools-extra/clangd/Preamble.h index 160b884beb56bb7..be8fed4ab88cdd8 100644 --- a/clang-tools-extra/clangd/Preamble.h +++ b/clang-tools-extra/clangd/Preamble.h @@ -27,6 +27,8 @@ #include "Diagnostics.h" #include "FS.h" #include "Headers.h" +#include "ModulesBuilder.h" + #include "clang-include-cleaner/Record.h" #include "support/Path.h" #include "clang/Basic/SourceManager.h" @@ -109,6 +111,8 @@ struct PreambleData { IncludeStructure Includes; // Captures #include-mapping information in #included headers. std::shared_ptr Pragmas; + // Information about required module files for this preamble. + std::unique_ptr RequiredModules; // Macros defined in the preamble section of the main file. // Users care about headers vs main-file, not preamble vs non-preamble. // These should be treated as main-file entities e.g. for code completion. diff --git a/clang-tools-extra/clangd/ProjectModules.h b/clang-tools-extra/clangd/ProjectModules.h new file mode 100644 index 000000000000000..3b9b564a87da015 --- /dev/null +++ b/clang-tools-extra/clangd/ProjectModules.h @@ -0,0 +1,50 @@ +//===------------------ ProjectModules.h -------------------------*- C++-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_PROJECTMODULES_H +#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_PROJECTMODULES_H + +#include "support/Path.h" +#include "support/ThreadsafeFS.h" + +#include + +namespace clang { +namespace clangd { + +/// An interface to query the modules information in the project. +/// Users should get instances of `ProjectModules` from +/// `GlobalCompilationDatabase::getProjectModules(PathRef)`. +/// +/// Currently, the modules information includes: +/// - Given a source file, what are the required modules. +/// - Given a module name and a required source file, what is +/// the corresponding source file. +/// +/// Note that there can be multiple source files declaring the same module +/// in a valid project. Although the language specification requires that +/// every module unit's name must be unique in valid program, there can be +/// multiple program in a project. And it is technically valid if these program +/// doesn't interfere with each other. +/// +/// A module name should be in the format: +/// `[:partition-name]`. So module names covers partitions. +class ProjectModules { +public: + virtual std::vector getRequiredModules(PathRef File) = 0; + virtual PathRef + getSourceForModuleName(llvm::StringRef ModuleName, + PathRef RequiredSrcFile = PathRef()) = 0; + + virtual ~ProjectModules() = default; +}; + +} // namespace clangd +} // namespace clang + +#endif diff --git a/clang-tools-extra/clangd/ScanningProjectModules.cpp b/clang-tools-extra/clangd/ScanningProjectModules.cpp new file mode 100644 index 000000000000000..92f75ef7d5c25ad --- /dev/null +++ b/clang-tools-extra/clangd/ScanningProjectModules.cpp @@ -0,0 +1,202 @@ +//===------------------ ProjectModules.h -------------------------*- C++-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "ProjectModules.h" +#include "support/Logger.h" +#include "clang/Tooling/DependencyScanning/DependencyScanningService.h" +#include "clang/Tooling/DependencyScanning/DependencyScanningTool.h" + +namespace clang::clangd { +namespace { +/// A scanner to query the dependency information for C++20 Modules. +/// +/// The scanner can scan a single file with `scan(PathRef)` member function +/// or scan the whole project with `globalScan(vector)` member +/// function. See the comments of `globalScan` to see the details. +/// +/// The ModuleDependencyScanner can get the directly required module names for a +/// specific source file. Also the ModuleDependencyScanner can get the source +/// file declaring the primary module interface for a specific module name. +/// +/// IMPORTANT NOTE: we assume that every module unit is only declared once in a +/// source file in the project. But the assumption is not strictly true even +/// besides the invalid projects. The language specification requires that every +/// module unit should be unique in a valid program. But a project can contain +/// multiple programs. Then it is valid that we can have multiple source files +/// declaring the same module in a project as long as these source files don't +/// interfere with each other. +class ModuleDependencyScanner { +public: + ModuleDependencyScanner( + std::shared_ptr CDB, + const ThreadsafeFS &TFS) + : CDB(CDB), TFS(TFS), + Service(tooling::dependencies::ScanningMode::CanonicalPreprocessing, + tooling::dependencies::ScanningOutputFormat::P1689) {} + + /// The scanned modules dependency information for a specific source file. + struct ModuleDependencyInfo { + /// The name of the module if the file is a module unit. + std::optional ModuleName; + /// A list of names for the modules that the file directly depends. + std::vector RequiredModules; + }; + + /// Scanning the single file specified by \param FilePath. + std::optional scan(PathRef FilePath); + + /// Scanning every source file in the current project to get the + /// to map. + /// TODO: We should find an efficient method to get the + /// to map. We can make it either by providing + /// a global module dependency scanner to monitor every file. Or we + /// can simply require the build systems (or even the end users) + /// to provide the map. + void globalScan(); + + /// Get the source file from the module name. Note that the language + /// guarantees all the module names are unique in a valid program. + /// This function should only be called after globalScan. + /// + /// TODO: We should handle the case that there are multiple source files + /// declaring the same module. + PathRef getSourceForModuleName(llvm::StringRef ModuleName) const; + + /// Return the direct required modules. Indirect required modules are not + /// included. + std::vector getRequiredModules(PathRef File); + +private: + std::shared_ptr CDB; + const ThreadsafeFS &TFS; + + // Whether the scanner has scanned the project globally. + bool GlobalScanned = false; + + clang::tooling::dependencies::DependencyScanningService Service; + + // TODO: Add a scanning cache. + + // Map module name to source file path. + llvm::StringMap ModuleNameToSource; +}; + +std::optional +ModuleDependencyScanner::scan(PathRef FilePath) { + auto Candidates = CDB->getCompileCommands(FilePath); + if (Candidates.empty()) + return std::nullopt; + + // Choose the first candidates as the compile commands as the file. + // Following the same logic with + // DirectoryBasedGlobalCompilationDatabase::getCompileCommand. + tooling::CompileCommand Cmd = std::move(Candidates.front()); + + static int StaticForMainAddr; // Just an address in this process. + Cmd.CommandLine.push_back("-resource-dir=" + + CompilerInvocation::GetResourcesPath( + "clangd", (void *)&StaticForMainAddr)); + + using namespace clang::tooling::dependencies; + + llvm::SmallString<128> FilePathDir(FilePath); + llvm::sys::path::remove_filename(FilePathDir); + DependencyScanningTool ScanningTool(Service, TFS.view(FilePathDir)); + + llvm::Expected ScanningResult = + ScanningTool.getP1689ModuleDependencyFile(Cmd, Cmd.Directory); + + if (auto E = ScanningResult.takeError()) { + elog("Scanning modules dependencies for {0} failed: {1}", FilePath, + llvm::toString(std::move(E))); + return std::nullopt; + } + + ModuleDependencyInfo Result; + + if (ScanningResult->Provides) { + ModuleNameToSource[ScanningResult->Provides->ModuleName] = FilePath; + Result.ModuleName = ScanningResult->Provides->ModuleName; + } + + for (auto &Required : ScanningResult->Requires) + Result.RequiredModules.push_back(Required.ModuleName); + + return Result; +} + +void ModuleDependencyScanner::globalScan() { + for (auto &File : CDB->getAllFiles()) + scan(File); + + GlobalScanned = true; +} + +PathRef ModuleDependencyScanner::getSourceForModuleName( + llvm::StringRef ModuleName) const { + assert( + GlobalScanned && + "We should only call getSourceForModuleName after calling globalScan()"); + + if (auto It = ModuleNameToSource.find(ModuleName); + It != ModuleNameToSource.end()) + return It->second; + + return {}; +} + +std::vector +ModuleDependencyScanner::getRequiredModules(PathRef File) { + auto ScanningResult = scan(File); + if (!ScanningResult) + return {}; + + return ScanningResult->RequiredModules; +} +} // namespace + +/// TODO: The existing `ScanningAllProjectModules` is not efficient. See the +/// comments in ModuleDependencyScanner for detail. +/// +/// In the future, we wish the build system can provide a well design +/// compilation database for modules then we can query that new compilation +/// database directly. Or we need to have a global long-live scanner to detect +/// the state of each file. +class ScanningAllProjectModules : public ProjectModules { +public: + ScanningAllProjectModules( + std::shared_ptr CDB, + const ThreadsafeFS &TFS) + : Scanner(CDB, TFS) {} + + ~ScanningAllProjectModules() override = default; + + std::vector getRequiredModules(PathRef File) override { + return Scanner.getRequiredModules(File); + } + + /// RequiredSourceFile is not used intentionally. See the comments of + /// ModuleDependencyScanner for detail. + PathRef + getSourceForModuleName(llvm::StringRef ModuleName, + PathRef RequiredSourceFile = PathRef()) override { + Scanner.globalScan(); + return Scanner.getSourceForModuleName(ModuleName); + } + +private: + ModuleDependencyScanner Scanner; +}; + +std::unique_ptr scanningProjectModules( + std::shared_ptr CDB, + const ThreadsafeFS &TFS) { + return std::make_unique(CDB, TFS); +} + +} // namespace clang::clangd diff --git a/clang-tools-extra/clangd/ScanningProjectModules.h b/clang-tools-extra/clangd/ScanningProjectModules.h new file mode 100644 index 000000000000000..75fc7dbcebce540 --- /dev/null +++ b/clang-tools-extra/clangd/ScanningProjectModules.h @@ -0,0 +1,26 @@ +//===------------ ScanningProjectModules.h -----------------------*- C++-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_SCANNINGPROJECTMODULES_H +#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_SCANNINGPROJECTMODULES_H + +#include "ProjectModules.h" +#include "clang/Tooling/CompilationDatabase.h" + +namespace clang { +namespace clangd { + +/// Providing modules information for the project by scanning every file. +std::unique_ptr scanningProjectModules( + std::shared_ptr CDB, + const ThreadsafeFS &TFS); + +} // namespace clangd +} // namespace clang + +#endif diff --git a/clang-tools-extra/clangd/test/CMakeLists.txt b/clang-tools-extra/clangd/test/CMakeLists.txt index d073267066e0b4c..b51f461a4986659 100644 --- a/clang-tools-extra/clangd/test/CMakeLists.txt +++ b/clang-tools-extra/clangd/test/CMakeLists.txt @@ -2,6 +2,7 @@ set(CLANGD_TEST_DEPS clangd ClangdTests clangd-indexer + split-file # No tests for it, but we should still make sure they build. dexp ) diff --git a/clang-tools-extra/clangd/test/modules.test b/clang-tools-extra/clangd/test/modules.test new file mode 100644 index 000000000000000..74280811a6cff84 --- /dev/null +++ b/clang-tools-extra/clangd/test/modules.test @@ -0,0 +1,83 @@ +# A smoke test to check the modules can work basically. +# +# Windows have different escaping modes. +# FIXME: We should add one for windows. +# UNSUPPORTED: system-windows +# +# RUN: rm -fr %t +# RUN: mkdir -p %t +# RUN: split-file %s %t +# +# RUN: sed -e "s|DIR|%/t|g" %t/compile_commands.json.tmpl > %t/compile_commands.json.tmp +# RUN: sed -e "s|CLANG_CC|%clang|g" %t/compile_commands.json.tmp > %t/compile_commands.json +# RUN: sed -e "s|DIR|%/t|g" %t/definition.jsonrpc.tmpl > %t/definition.jsonrpc +# +# RUN: clangd -experimental-modules-support -lit-test < %t/definition.jsonrpc \ +# RUN: | FileCheck -strict-whitespace %t/definition.jsonrpc + +#--- A.cppm +export module A; +export void printA() {} + +#--- Use.cpp +import A; +void foo() { + print +} + +#--- compile_commands.json.tmpl +[ + { + "directory": "DIR", + "command": "CLANG_CC -fprebuilt-module-path=DIR -std=c++20 -o DIR/main.cpp.o -c DIR/Use.cpp", + "file": "DIR/Use.cpp" + }, + { + "directory": "DIR", + "command": "CLANG_CC -std=c++20 DIR/A.cppm --precompile -o DIR/A.pcm", + "file": "DIR/A.cppm" + } +] + +#--- definition.jsonrpc.tmpl +{ + "jsonrpc": "2.0", + "id": 0, + "method": "initialize", + "params": { + "processId": 123, + "rootPath": "clangd", + "capabilities": { + "textDocument": { + "completion": { + "completionItem": { + "snippetSupport": true + } + } + } + }, + "trace": "off" + } +} +--- +{ + "jsonrpc": "2.0", + "method": "textDocument/didOpen", + "params": { + "textDocument": { + "uri": "file://DIR/Use.cpp", + "languageId": "cpp", + "version": 1, + "text": "import A;\nvoid foo() {\n print\n}\n" + } + } +} + +# CHECK: "message"{{.*}}printA{{.*}}(fix available) + +--- +{"jsonrpc":"2.0","id":1,"method":"textDocument/completion","params":{"textDocument":{"uri":"file://DIR/Use.cpp"},"context":{"triggerKind":1},"position":{"line":2,"character":6}}} +--- +{"jsonrpc":"2.0","id":2,"method":"shutdown"} +--- +{"jsonrpc":"2.0","method":"exit"} diff --git a/clang-tools-extra/clangd/tool/Check.cpp b/clang-tools-extra/clangd/tool/Check.cpp index 25005ec1bd04532..bc2eaa77a66eeca 100644 --- a/clang-tools-extra/clangd/tool/Check.cpp +++ b/clang-tools-extra/clangd/tool/Check.cpp @@ -146,10 +146,13 @@ class Checker { ClangdLSPServer::Options Opts; // from buildCommand tooling::CompileCommand Cmd; + std::unique_ptr BaseCDB; + std::unique_ptr CDB; // from buildInvocation ParseInputs Inputs; std::unique_ptr Invocation; format::FormatStyle Style; + std::optional ModulesManager; // from buildAST std::shared_ptr Preamble; std::optional AST; @@ -168,14 +171,14 @@ class Checker { DirectoryBasedGlobalCompilationDatabase::Options CDBOpts(TFS); CDBOpts.CompileCommandsDir = Config::current().CompileFlags.CDBSearch.FixedCDBPath; - std::unique_ptr BaseCDB = + BaseCDB = std::make_unique(CDBOpts); auto Mangler = CommandMangler::detect(); Mangler.SystemIncludeExtractor = getSystemIncludeExtractor(llvm::ArrayRef(Opts.QueryDriverGlobs)); if (Opts.ResourceDir) Mangler.ResourceDir = *Opts.ResourceDir; - auto CDB = std::make_unique( + CDB = std::make_unique( BaseCDB.get(), std::vector{}, std::move(Mangler)); if (auto TrueCmd = CDB->getCompileCommand(File)) { @@ -213,6 +216,11 @@ class Checker { return false; } } + if (Opts.EnableExperimentalModulesSupport) { + if (!ModulesManager) + ModulesManager.emplace(*CDB); + Inputs.ModulesManager = &*ModulesManager; + } log("Parsing command..."); Invocation = buildCompilerInvocation(Inputs, CaptureInvocationDiags, &CC1Args); diff --git a/clang-tools-extra/clangd/tool/ClangdMain.cpp b/clang-tools-extra/clangd/tool/ClangdMain.cpp index 73000d96c6ca8e5..3a5449ac8c79944 100644 --- a/clang-tools-extra/clangd/tool/ClangdMain.cpp +++ b/clang-tools-extra/clangd/tool/ClangdMain.cpp @@ -551,6 +551,13 @@ opt ProjectRoot{ }; #endif +opt ExperimentalModulesSupport{ + "experimental-modules-support", + cat(Features), + desc("Experimental support for standard c++ modules"), + init(false), +}; + /// Supports a test URI scheme with relaxed constraints for lit tests. /// The path in a test URI will be combined with a platform-specific fake /// directory to form an absolute path. For example, test:///a.cpp is resolved @@ -860,6 +867,7 @@ clangd accepts flags on the commandline, and in the CLANGD_FLAGS environment var ClangdLSPServer::Options Opts; Opts.UseDirBasedCDB = (CompileArgsFrom == FilesystemCompileArgs); + Opts.EnableExperimentalModulesSupport = ExperimentalModulesSupport; switch (PCHStorage) { case PCHStorageFlag::Memory: diff --git a/clang-tools-extra/clangd/unittests/CMakeLists.txt b/clang-tools-extra/clangd/unittests/CMakeLists.txt index 0d4628ccf25d8c9..4fa9f18407ae9e6 100644 --- a/clang-tools-extra/clangd/unittests/CMakeLists.txt +++ b/clang-tools-extra/clangd/unittests/CMakeLists.txt @@ -74,6 +74,7 @@ add_unittest(ClangdUnitTests ClangdTests LoggerTests.cpp LSPBinderTests.cpp LSPClient.cpp + PrerequisiteModulesTest.cpp ModulesTests.cpp ParsedASTTests.cpp PathMappingTests.cpp diff --git a/clang-tools-extra/clangd/unittests/PrerequisiteModulesTest.cpp b/clang-tools-extra/clangd/unittests/PrerequisiteModulesTest.cpp new file mode 100644 index 000000000000000..7bbb95c8b8d67f4 --- /dev/null +++ b/clang-tools-extra/clangd/unittests/PrerequisiteModulesTest.cpp @@ -0,0 +1,408 @@ +//===--------------- PrerequisiteModulesTests.cpp -------------------*- C++ +//-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +/// FIXME: Skip testing on windows temporarily due to the different escaping +/// code mode. +#ifndef _WIN32 + +#include "ModulesBuilder.h" +#include "ScanningProjectModules.h" +#include "Annotations.h" +#include "CodeComplete.h" +#include "Compiler.h" +#include "TestTU.h" +#include "support/ThreadsafeFS.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/raw_ostream.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace clang::clangd { +namespace { + +class MockDirectoryCompilationDatabase : public MockCompilationDatabase { +public: + MockDirectoryCompilationDatabase(StringRef TestDir, const ThreadsafeFS &TFS) + : MockCompilationDatabase(TestDir), + MockedCDBPtr(std::make_shared(*this)), + TFS(TFS) { + this->ExtraClangFlags.push_back("-std=c++20"); + this->ExtraClangFlags.push_back("-c"); + } + + void addFile(llvm::StringRef Path, llvm::StringRef Contents); + + std::unique_ptr getProjectModules(PathRef) const override { + return scanningProjectModules(MockedCDBPtr, TFS); + } + +private: + class MockClangCompilationDatabase : public tooling::CompilationDatabase { + public: + MockClangCompilationDatabase(MockDirectoryCompilationDatabase &MCDB) + : MCDB(MCDB) {} + + std::vector + getCompileCommands(StringRef FilePath) const override { + std::optional Cmd = + MCDB.getCompileCommand(FilePath); + EXPECT_TRUE(Cmd); + return {*Cmd}; + } + + std::vector getAllFiles() const override { return Files; } + + void AddFile(StringRef File) { Files.push_back(File.str()); } + + private: + MockDirectoryCompilationDatabase &MCDB; + std::vector Files; + }; + + std::shared_ptr MockedCDBPtr; + const ThreadsafeFS &TFS; +}; + +// Add files to the working testing directory and the compilation database. +void MockDirectoryCompilationDatabase::addFile(llvm::StringRef Path, + llvm::StringRef Contents) { + ASSERT_FALSE(llvm::sys::path::is_absolute(Path)); + + SmallString<256> AbsPath(Directory); + llvm::sys::path::append(AbsPath, Path); + + ASSERT_FALSE( + llvm::sys::fs::create_directories(llvm::sys::path::parent_path(AbsPath))); + + std::error_code EC; + llvm::raw_fd_ostream OS(AbsPath, EC); + ASSERT_FALSE(EC); + OS << Contents; + + MockedCDBPtr->AddFile(Path); +} + +class PrerequisiteModulesTests : public ::testing::Test { +protected: + void SetUp() override { + ASSERT_FALSE(llvm::sys::fs::createUniqueDirectory("modules-test", TestDir)); + } + + void TearDown() override { + ASSERT_FALSE(llvm::sys::fs::remove_directories(TestDir)); + } + +public: + // Get the absolute path for file specified by Path under testing working + // directory. + std::string getFullPath(llvm::StringRef Path) { + SmallString<128> Result(TestDir); + llvm::sys::path::append(Result, Path); + EXPECT_TRUE(llvm::sys::fs::exists(Result.str())); + return Result.str().str(); + } + + ParseInputs getInputs(llvm::StringRef FileName, + const GlobalCompilationDatabase &CDB) { + std::string FullPathName = getFullPath(FileName); + + ParseInputs Inputs; + std::optional Cmd = + CDB.getCompileCommand(FullPathName); + EXPECT_TRUE(Cmd); + Inputs.CompileCommand = std::move(*Cmd); + Inputs.TFS = &FS; + + if (auto Contents = FS.view(TestDir)->getBufferForFile(FullPathName)) + Inputs.Contents = Contents->get()->getBuffer().str(); + + return Inputs; + } + + SmallString<256> TestDir; + // FIXME: It will be better to use the MockFS if the scanning process and + // build module process doesn't depend on reading real IO. + RealThreadsafeFS FS; + + DiagnosticConsumer DiagConsumer; +}; + +TEST_F(PrerequisiteModulesTests, NonModularTest) { + MockDirectoryCompilationDatabase CDB(TestDir, FS); + + CDB.addFile("foo.h", R"cpp( +inline void foo() {} + )cpp"); + + CDB.addFile("NonModular.cpp", R"cpp( +#include "foo.h" +void use() { + foo(); +} + )cpp"); + + ModulesBuilder Builder(CDB); + + // NonModular.cpp is not related to modules. So nothing should be built. + auto NonModularInfo = + Builder.buildPrerequisiteModulesFor(getFullPath("NonModular.cpp"), FS); + EXPECT_TRUE(NonModularInfo); + + HeaderSearchOptions HSOpts; + NonModularInfo->adjustHeaderSearchOptions(HSOpts); + EXPECT_TRUE(HSOpts.PrebuiltModuleFiles.empty()); + + auto Invocation = + buildCompilerInvocation(getInputs("NonModular.cpp", CDB), DiagConsumer); + EXPECT_TRUE(NonModularInfo->canReuse(*Invocation, FS.view(TestDir))); +} + +TEST_F(PrerequisiteModulesTests, ModuleWithoutDepTest) { + MockDirectoryCompilationDatabase CDB(TestDir, FS); + + CDB.addFile("foo.h", R"cpp( +inline void foo() {} + )cpp"); + + CDB.addFile("M.cppm", R"cpp( +module; +#include "foo.h" +export module M; + )cpp"); + + ModulesBuilder Builder(CDB); + + auto MInfo = Builder.buildPrerequisiteModulesFor(getFullPath("M.cppm"), FS); + EXPECT_TRUE(MInfo); + + // Nothing should be built since M doesn't dependent on anything. + HeaderSearchOptions HSOpts; + MInfo->adjustHeaderSearchOptions(HSOpts); + EXPECT_TRUE(HSOpts.PrebuiltModuleFiles.empty()); + + auto Invocation = + buildCompilerInvocation(getInputs("M.cppm", CDB), DiagConsumer); + EXPECT_TRUE(MInfo->canReuse(*Invocation, FS.view(TestDir))); +} + +TEST_F(PrerequisiteModulesTests, ModuleWithDepTest) { + MockDirectoryCompilationDatabase CDB(TestDir, FS); + + CDB.addFile("foo.h", R"cpp( +inline void foo() {} + )cpp"); + + CDB.addFile("M.cppm", R"cpp( +module; +#include "foo.h" +export module M; + )cpp"); + + CDB.addFile("N.cppm", R"cpp( +export module N; +import :Part; +import M; + )cpp"); + + CDB.addFile("N-part.cppm", R"cpp( +// Different module name with filename intentionally. +export module N:Part; + )cpp"); + + ModulesBuilder Builder(CDB); + + auto NInfo = Builder.buildPrerequisiteModulesFor(getFullPath("N.cppm"), FS); + EXPECT_TRUE(NInfo); + + ParseInputs NInput = getInputs("N.cppm", CDB); + std::unique_ptr Invocation = + buildCompilerInvocation(NInput, DiagConsumer); + // Test that `PrerequisiteModules::canReuse` works basically. + EXPECT_TRUE(NInfo->canReuse(*Invocation, FS.view(TestDir))); + + { + // Check that + // `PrerequisiteModules::adjustHeaderSearchOptions(HeaderSearchOptions&)` + // can appending HeaderSearchOptions correctly. + HeaderSearchOptions HSOpts; + NInfo->adjustHeaderSearchOptions(HSOpts); + + EXPECT_TRUE(HSOpts.PrebuiltModuleFiles.count("M")); + EXPECT_TRUE(HSOpts.PrebuiltModuleFiles.count("N:Part")); + } + + { + // Check that + // `PrerequisiteModules::adjustHeaderSearchOptions(HeaderSearchOptions&)` + // can replace HeaderSearchOptions correctly. + HeaderSearchOptions HSOpts; + HSOpts.PrebuiltModuleFiles["M"] = "incorrect_path"; + HSOpts.PrebuiltModuleFiles["N:Part"] = "incorrect_path"; + NInfo->adjustHeaderSearchOptions(HSOpts); + + EXPECT_TRUE(StringRef(HSOpts.PrebuiltModuleFiles["M"]).ends_with(".pcm")); + EXPECT_TRUE( + StringRef(HSOpts.PrebuiltModuleFiles["N:Part"]).ends_with(".pcm")); + } +} + +TEST_F(PrerequisiteModulesTests, ReusabilityTest) { + MockDirectoryCompilationDatabase CDB(TestDir, FS); + + CDB.addFile("foo.h", R"cpp( +inline void foo() {} + )cpp"); + + CDB.addFile("M.cppm", R"cpp( +module; +#include "foo.h" +export module M; + )cpp"); + + CDB.addFile("N.cppm", R"cpp( +export module N; +import :Part; +import M; + )cpp"); + + CDB.addFile("N-part.cppm", R"cpp( +// Different module name with filename intentionally. +export module N:Part; + )cpp"); + + ModulesBuilder Builder(CDB); + + auto NInfo = Builder.buildPrerequisiteModulesFor(getFullPath("N.cppm"), FS); + EXPECT_TRUE(NInfo); + EXPECT_TRUE(NInfo); + + ParseInputs NInput = getInputs("N.cppm", CDB); + std::unique_ptr Invocation = + buildCompilerInvocation(NInput, DiagConsumer); + EXPECT_TRUE(NInfo->canReuse(*Invocation, FS.view(TestDir))); + + // Test that we can still reuse the NInfo after we touch a unrelated file. + { + CDB.addFile("L.cppm", R"cpp( +module; +#include "foo.h" +export module L; +export int ll = 43; + )cpp"); + EXPECT_TRUE(NInfo->canReuse(*Invocation, FS.view(TestDir))); + + CDB.addFile("bar.h", R"cpp( +inline void bar() {} +inline void bar(int) {} + )cpp"); + EXPECT_TRUE(NInfo->canReuse(*Invocation, FS.view(TestDir))); + } + + // Test that we can't reuse the NInfo after we touch a related file. + { + CDB.addFile("M.cppm", R"cpp( +module; +#include "foo.h" +export module M; +export int mm = 44; + )cpp"); + EXPECT_FALSE(NInfo->canReuse(*Invocation, FS.view(TestDir))); + + NInfo = Builder.buildPrerequisiteModulesFor(getFullPath("N.cppm"), FS); + EXPECT_TRUE(NInfo->canReuse(*Invocation, FS.view(TestDir))); + + CDB.addFile("foo.h", R"cpp( +inline void foo() {} +inline void foo(int) {} + )cpp"); + EXPECT_FALSE(NInfo->canReuse(*Invocation, FS.view(TestDir))); + + NInfo = Builder.buildPrerequisiteModulesFor(getFullPath("N.cppm"), FS); + EXPECT_TRUE(NInfo->canReuse(*Invocation, FS.view(TestDir))); + } + + CDB.addFile("N-part.cppm", R"cpp( +export module N:Part; +// Intentioned to make it uncompilable. +export int NPart = 4LIdjwldijaw + )cpp"); + EXPECT_FALSE(NInfo->canReuse(*Invocation, FS.view(TestDir))); + NInfo = Builder.buildPrerequisiteModulesFor(getFullPath("N.cppm"), FS); + EXPECT_TRUE(NInfo); + EXPECT_FALSE(NInfo->canReuse(*Invocation, FS.view(TestDir))); + + CDB.addFile("N-part.cppm", R"cpp( +export module N:Part; +export int NPart = 43; + )cpp"); + EXPECT_TRUE(NInfo); + EXPECT_FALSE(NInfo->canReuse(*Invocation, FS.view(TestDir))); + NInfo = Builder.buildPrerequisiteModulesFor(getFullPath("N.cppm"), FS); + EXPECT_TRUE(NInfo); + EXPECT_TRUE(NInfo->canReuse(*Invocation, FS.view(TestDir))); + + // Test that if we changed the modification time of the file, the module files + // info is still reusable if its content doesn't change. + CDB.addFile("N-part.cppm", R"cpp( +export module N:Part; +export int NPart = 43; + )cpp"); + EXPECT_TRUE(NInfo->canReuse(*Invocation, FS.view(TestDir))); + + CDB.addFile("N.cppm", R"cpp( +export module N; +import :Part; +import M; + +export int nn = 43; + )cpp"); + // NInfo should be reusable after we change its content. + EXPECT_TRUE(NInfo->canReuse(*Invocation, FS.view(TestDir))); +} + +// An End-to-End test for modules. +TEST_F(PrerequisiteModulesTests, ParsedASTTest) { + MockDirectoryCompilationDatabase CDB(TestDir, FS); + + CDB.addFile("A.cppm", R"cpp( +export module A; +export void printA(); + )cpp"); + + CDB.addFile("Use.cpp", R"cpp( +import A; +)cpp"); + + ModulesBuilder Builder(CDB); + + ParseInputs Use = getInputs("Use.cpp", CDB); + Use.ModulesManager = &Builder; + + std::unique_ptr CI = + buildCompilerInvocation(Use, DiagConsumer); + EXPECT_TRUE(CI); + + auto Preamble = + buildPreamble(getFullPath("Use.cpp"), *CI, Use, /*InMemory=*/true, + /*Callback=*/nullptr); + EXPECT_TRUE(Preamble); + EXPECT_TRUE(Preamble->RequiredModules); + + auto AST = ParsedAST::build(getFullPath("Use.cpp"), Use, std::move(CI), {}, + Preamble); + EXPECT_TRUE(AST); + + const NamedDecl &D = findDecl(*AST, "printA"); + EXPECT_TRUE(D.isFromASTFile()); +} + +} // namespace +} // namespace clang::clangd + +#endif diff --git a/clang-tools-extra/clangd/unittests/TestFS.h b/clang-tools-extra/clangd/unittests/TestFS.h index 6bdadc9c07439d8..568533f3b3b9114 100644 --- a/clang-tools-extra/clangd/unittests/TestFS.h +++ b/clang-tools-extra/clangd/unittests/TestFS.h @@ -67,7 +67,7 @@ class MockCompilationDatabase : public GlobalCompilationDatabase { std::vector ExtraClangFlags; -private: +protected: StringRef Directory; StringRef RelPathPrefix; }; diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst index 004811d2eca4f49..697b514ae1572ac 100644 --- a/clang-tools-extra/docs/ReleaseNotes.rst +++ b/clang-tools-extra/docs/ReleaseNotes.rst @@ -48,6 +48,10 @@ Major New Features Improvements to clangd ---------------------- +- Introduced exmperimental support for C++20 Modules. The experimental support can + be enabled by `-experimental-modules-support` option. It is in an early development + stage and may not perform efficiently in real-world scenarios. + Inlay hints ^^^^^^^^^^^