versioner: use a virtual filesystem for input files.
Use an InMemoryFileSystem to store and share input files across
compilations.
This improves the result of `time versioner` further, from:
versioner 109.12s user 17.43s system 2433% cpu 5.201 total
to:
versioner 112.20s user 1.38s system 2416% cpu 4.700 total
Bug: http://b/32748936
Test: python run_tests.py
Change-Id: I72d37b7c30850b8399cc40338247700fe3e7b2f9
diff --git a/tools/versioner/src/Android.mk b/tools/versioner/src/Android.mk
index 713d6c1..c90e02f 100644
--- a/tools/versioner/src/Android.mk
+++ b/tools/versioner/src/Android.mk
@@ -31,7 +31,8 @@
Driver.cpp \
Preprocessor.cpp \
SymbolDatabase.cpp \
- Utils.cpp
+ Utils.cpp \
+ VFS.cpp
LOCAL_SHARED_LIBRARIES := libclang libLLVM libbase
diff --git a/tools/versioner/src/Driver.cpp b/tools/versioner/src/Driver.cpp
index dad8a1e..215dc3c 100644
--- a/tools/versioner/src/Driver.cpp
+++ b/tools/versioner/src/Driver.cpp
@@ -29,6 +29,7 @@
#include <clang/AST/ASTConsumer.h>
#include <clang/Basic/Diagnostic.h>
#include <clang/Basic/TargetInfo.h>
+#include <clang/Basic/VirtualFileSystem.h>
#include <clang/Driver/Compilation.h>
#include <clang/Driver/Driver.h>
#include <clang/Frontend/CompilerInstance.h>
@@ -92,7 +93,8 @@
// Run it once to generate flags for each target, and memoize the results.
static std::unordered_map<CompilationType, std::vector<std::string>> cc1_flags;
static const char* filename_placeholder = "__VERSIONER_PLACEHOLDER__";
-static void generateTargetCC1Flags(CompilationType type,
+static void generateTargetCC1Flags(llvm::IntrusiveRefCntPtr<clang::vfs::FileSystem> vfs,
+ CompilationType type,
const std::vector<std::string>& include_dirs) {
std::vector<std::string> cmd = { "versioner" };
cmd.push_back("-std=c11");
@@ -122,12 +124,8 @@
cmd.push_back("-nostdinc");
if (add_include) {
- const char* top = getenv("ANDROID_BUILD_TOP");
- if (!top) {
- errx(1, "-i passed, but ANDROID_BUILD_TOP is unset");
- }
cmd.push_back("-include");
- cmd.push_back(to_string(top) + "/bionic/libc/include/android/versioning.h");
+ cmd.push_back("android/versioning.h");
}
for (const auto& dir : include_dirs) {
@@ -138,7 +136,7 @@
cmd.push_back(filename_placeholder);
auto diags = constructDiags();
- driver::Driver driver("versioner", llvm::sys::getDefaultTargetTriple(), *diags);
+ driver::Driver driver("versioner", llvm::sys::getDefaultTargetTriple(), *diags, vfs);
driver.setCheckInputsExist(false);
llvm::SmallVector<const char*, 32> driver_args;
@@ -192,7 +190,8 @@
return result;
}
-void initializeTargetCC1FlagCache(const std::set<CompilationType>& types,
+void initializeTargetCC1FlagCache(llvm::IntrusiveRefCntPtr<clang::vfs::FileSystem> vfs,
+ const std::set<CompilationType>& types,
const std::unordered_map<Arch, CompilationRequirements>& reqs) {
if (!cc1_flags.empty()) {
errx(1, "reinitializing target CC1 flag cache?");
@@ -201,14 +200,14 @@
auto start = std::chrono::high_resolution_clock::now();
std::vector<std::thread> threads;
for (const CompilationType type : types) {
- threads.emplace_back([type, &reqs]() {
+ threads.emplace_back([type, &vfs, &reqs]() {
const auto& arch_req_it = reqs.find(type.arch);
if (arch_req_it == reqs.end()) {
errx(1, "CompilationRequirement map missing entry for CompilationType %s",
to_string(type).c_str());
}
- generateTargetCC1Flags(type, arch_req_it->second.dependencies);
+ generateTargetCC1Flags(vfs, type, arch_req_it->second.dependencies);
});
}
for (auto& thread : threads) {
@@ -226,7 +225,8 @@
}
}
-void compileHeader(HeaderDatabase* header_database, CompilationType type,
+void compileHeader(llvm::IntrusiveRefCntPtr<clang::vfs::FileSystem> vfs,
+ HeaderDatabase* header_database, CompilationType type,
const std::string& filename) {
auto diags = constructDiags();
std::vector<const char*> cc1_flags = getCC1Command(type, filename);
@@ -239,6 +239,7 @@
clang::CompilerInstance Compiler;
Compiler.setInvocation(invocation.release());
Compiler.setDiagnostics(diags.get());
+ Compiler.setVirtualFileSystem(vfs);
VersionerASTAction versioner_action(header_database, type);
if (!Compiler.ExecuteAction(versioner_action)) {
diff --git a/tools/versioner/src/Driver.h b/tools/versioner/src/Driver.h
index c1a4b24..48683e4 100644
--- a/tools/versioner/src/Driver.h
+++ b/tools/versioner/src/Driver.h
@@ -20,16 +20,21 @@
#include <string>
#include <unordered_map>
+#include <llvm/ADT/IntrusiveRefCntPtr.h>
+
#include "Arch.h"
#include "DeclarationDatabase.h"
+#include "VFS.h"
struct CompilationRequirements {
std::vector<std::string> headers;
std::vector<std::string> dependencies;
};
-void initializeTargetCC1FlagCache(const std::set<CompilationType>& types,
+void initializeTargetCC1FlagCache(llvm::IntrusiveRefCntPtr<clang::vfs::FileSystem> vfs,
+ const std::set<CompilationType>& types,
const std::unordered_map<Arch, CompilationRequirements>& reqs);
-void compileHeader(HeaderDatabase* header_database, CompilationType type,
+void compileHeader(llvm::IntrusiveRefCntPtr<clang::vfs::FileSystem> vfs,
+ HeaderDatabase* header_database, CompilationType type,
const std::string& filename);
diff --git a/tools/versioner/src/VFS.cpp b/tools/versioner/src/VFS.cpp
new file mode 100644
index 0000000..cd6d367
--- /dev/null
+++ b/tools/versioner/src/VFS.cpp
@@ -0,0 +1,92 @@
+/*
+ * Copyright (C) 2016 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <err.h>
+#include <fcntl.h>
+#include <fts.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include <memory>
+#include <string>
+
+#include <android-base/unique_fd.h>
+#include <clang/Basic/VirtualFileSystem.h>
+#include <llvm/ADT/IntrusiveRefCntPtr.h>
+#include <llvm/Support/MemoryBuffer.h>
+
+#include "Utils.h"
+
+using android::base::unique_fd;
+using namespace clang::vfs;
+
+static void addDirectoryToVFS(InMemoryFileSystem* vfs, const std::string& path) {
+ char* paths[] = { const_cast<char*>(path.c_str()), nullptr };
+ FTS* fts = fts_open(paths, FTS_COMFOLLOW | FTS_LOGICAL | FTS_NOCHDIR, nullptr);
+ if (!fts) {
+ err(1, "failed to open directory %s", path.c_str());
+ }
+
+ while (FTSENT* ent = fts_read(fts)) {
+ if ((ent->fts_info & FTS_F) == 0) {
+ continue;
+ }
+
+ const char* file_path = ent->fts_accpath;
+ unique_fd fd(open(file_path, O_RDONLY | O_CLOEXEC));
+ if (fd == -1) {
+ err(1, "failed to open header '%s'", file_path);
+ }
+
+ auto buffer_opt = llvm::MemoryBuffer::getOpenFileSlice(fd, file_path, -1, 0);
+ if (!buffer_opt) {
+ errx(1, "failed to map header '%s'", file_path);
+ }
+
+ if (!vfs->addFile(file_path, ent->fts_statp->st_mtim.tv_sec, std::move(buffer_opt.get()))) {
+ errx(1, "failed to add file '%s'", file_path);
+ }
+ }
+
+ fts_close(fts);
+}
+
+llvm::IntrusiveRefCntPtr<FileSystem> createCommonVFS(const std::string& header_dir,
+ const std::string& dependency_dir,
+ bool add_versioning_header) {
+ auto vfs = std::make_unique<InMemoryFileSystem>();
+ addDirectoryToVFS(vfs.get(), header_dir);
+ if (!dependency_dir.empty()) {
+ addDirectoryToVFS(vfs.get(), dependency_dir);
+ }
+
+ if (add_versioning_header) {
+ const char* top = getenv("ANDROID_BUILD_TOP");
+ if (!top) {
+ errx(1, "-i passed, but ANDROID_BUILD_TOP is unset");
+ }
+
+ std::string header_path = std::string(top) + "/bionic/libc/include/android/versioning.h";
+ auto buffer_opt = llvm::MemoryBuffer::getFile(header_path);
+ if (!buffer_opt) {
+ err(1, "failed to open %s", header_path.c_str());
+ }
+ vfs->addFile("android/versioning.h", 0, std::move(buffer_opt.get()));
+ }
+
+ return llvm::IntrusiveRefCntPtr<FileSystem>(vfs.release());
+}
diff --git a/tools/versioner/src/VFS.h b/tools/versioner/src/VFS.h
new file mode 100644
index 0000000..e2ab002
--- /dev/null
+++ b/tools/versioner/src/VFS.h
@@ -0,0 +1,27 @@
+/*
+ * Copyright (C) 2016 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <memory>
+#include <string>
+
+#include <llvm/ADT/IntrusiveRefCntPtr.h>
+#include <clang/Basic/VirtualFileSystem.h>
+
+llvm::IntrusiveRefCntPtr<clang::vfs::FileSystem> createCommonVFS(const std::string& header_dir,
+ const std::string& dependency_dir,
+ bool add_versioning_header);
diff --git a/tools/versioner/src/versioner.cpp b/tools/versioner/src/versioner.cpp
index f0a2339..5aaf47a 100644
--- a/tools/versioner/src/versioner.cpp
+++ b/tools/versioner/src/versioner.cpp
@@ -44,6 +44,8 @@
#include "Preprocessor.h"
#include "SymbolDatabase.h"
#include "Utils.h"
+#include "VFS.h"
+
#include "versioner.h"
using namespace std::string_literals;
@@ -138,6 +140,8 @@
errx(1, "compileHeaders received no CompilationTypes");
}
+ auto vfs = createCommonVFS(header_dir, dependency_dir, add_include);
+
size_t thread_count = max_thread_count;
std::vector<std::thread> threads;
@@ -151,7 +155,7 @@
}
}
- initializeTargetCC1FlagCache(types, requirements);
+ initializeTargetCC1FlagCache(vfs, types, requirements);
std::vector<std::pair<CompilationType, const std::string&>> jobs;
for (CompilationType type : types) {
@@ -165,16 +169,16 @@
if (thread_count == 1) {
for (const auto& job : jobs) {
- compileHeader(result.get(), job.first, job.second);
+ compileHeader(vfs, result.get(), job.first, job.second);
}
} else {
// Spawn threads.
for (size_t i = 0; i < thread_count; ++i) {
- threads.emplace_back([&jobs, &result, &header_dir, thread_count, i]() {
+ threads.emplace_back([&jobs, &result, &header_dir, vfs, thread_count, i]() {
size_t index = i;
while (index < jobs.size()) {
const auto& job = jobs[index];
- compileHeader(result.get(), job.first, job.second);
+ compileHeader(vfs, result.get(), job.first, job.second);
index += thread_count;
}
});