versioner: introduce.
Add a clang-based tool to inspect header availability attributes and
verify them against the NDK platform definitions.
Bug: http://b/28178111
Change-Id: I1bb1925a620e98cc9606cb5a3360b1224c700bd0
diff --git a/tools/versioner/src/DeclarationDatabase.cpp b/tools/versioner/src/DeclarationDatabase.cpp
new file mode 100644
index 0000000..9f02588
--- /dev/null
+++ b/tools/versioner/src/DeclarationDatabase.cpp
@@ -0,0 +1,179 @@
+/*
+ * Copyright 2016 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "DeclarationDatabase.h"
+
+#include <iostream>
+#include <map>
+#include <set>
+#include <string>
+
+#include <clang/AST/AST.h>
+#include <clang/AST/Attr.h>
+#include <clang/AST/Mangle.h>
+#include <clang/AST/RecursiveASTVisitor.h>
+#include <clang/Frontend/ASTUnit.h>
+#include <llvm/Support/raw_ostream.h>
+
+using namespace clang;
+
+class Visitor : public RecursiveASTVisitor<Visitor> {
+ HeaderDatabase& database;
+ SourceManager& src_manager;
+ std::unique_ptr<MangleContext> mangler;
+
+ public:
+ Visitor(HeaderDatabase& database, ASTContext& ctx)
+ : database(database), src_manager(ctx.getSourceManager()) {
+ mangler.reset(ItaniumMangleContext::create(ctx, ctx.getDiagnostics()));
+ }
+
+ std::string getDeclName(NamedDecl* decl) {
+ if (auto var_decl = dyn_cast<VarDecl>(decl)) {
+ if (!var_decl->isFileVarDecl()) {
+ return "<local var>";
+ }
+ }
+
+ if (mangler->shouldMangleDeclName(decl)) {
+ std::string mangled;
+ llvm::raw_string_ostream ss(mangled);
+ mangler->mangleName(decl, ss);
+ return mangled;
+ }
+
+ auto identifier = decl->getIdentifier();
+ if (!identifier) {
+ return "<error>";
+ }
+ return identifier->getName();
+ }
+
+ bool VisitDecl(Decl* decl) {
+ // Skip declarations inside of functions (function arguments, variable declarations inside of
+ // inline functions, etc).
+ if (decl->getParentFunctionOrMethod()) {
+ return true;
+ }
+
+ auto named_decl = dyn_cast<NamedDecl>(decl);
+ if (!named_decl) {
+ return true;
+ }
+
+ DeclarationType declaration_type;
+ std::string declaration_name = getDeclName(named_decl);
+ bool is_extern = named_decl->getFormalLinkage() == ExternalLinkage;
+ bool is_definition = false;
+
+ if (auto function_decl = dyn_cast<FunctionDecl>(decl)) {
+ declaration_type = DeclarationType::function;
+ is_definition = function_decl->isThisDeclarationADefinition();
+ } else if (auto var_decl = dyn_cast<VarDecl>(decl)) {
+ if (!var_decl->isFileVarDecl()) {
+ return true;
+ }
+
+ declaration_type = DeclarationType::variable;
+ switch (var_decl->isThisDeclarationADefinition()) {
+ case VarDecl::DeclarationOnly:
+ is_definition = false;
+ break;
+
+ case VarDecl::Definition:
+ is_definition = true;
+ break;
+
+ case VarDecl::TentativeDefinition:
+ // Forbid tentative definitions in headers.
+ fprintf(stderr, "ERROR: declaration '%s' is a tentative definition\n",
+ declaration_name.c_str());
+ decl->dump();
+ abort();
+ }
+ } else {
+ // We only care about function and variable declarations.
+ return true;
+ }
+
+ if (decl->hasAttr<UnavailableAttr>()) {
+ // Skip declarations that exist only for compile-time diagnostics.
+ return true;
+ }
+
+ // Look for availability annotations.
+ DeclarationAvailability availability;
+ for (const AvailabilityAttr* attr : decl->specific_attrs<AvailabilityAttr>()) {
+ if (attr->getPlatform()->getName() != "android") {
+ fprintf(stderr, "skipping non-android platform %s\n",
+ attr->getPlatform()->getName().str().c_str());
+ continue;
+ }
+ if (attr->getIntroduced().getMajor() != 0) {
+ availability.introduced = attr->getIntroduced().getMajor();
+ }
+ if (attr->getDeprecated().getMajor() != 0) {
+ availability.deprecated = attr->getDeprecated().getMajor();
+ }
+ if (attr->getObsoleted().getMajor() != 0) {
+ availability.obsoleted = attr->getObsoleted().getMajor();
+ }
+ }
+
+ // Find or insert an entry for the declaration.
+ auto declaration_it = database.declarations.find(declaration_name);
+ if (declaration_it == database.declarations.end()) {
+ Declaration declaration = {.name = declaration_name };
+ bool inserted;
+ std::tie(declaration_it, inserted) =
+ database.declarations.insert({ declaration_name, declaration });
+ }
+
+ auto& declaration_locations = declaration_it->second.locations;
+ auto presumed_loc = src_manager.getPresumedLoc(decl->getLocation());
+ DeclarationLocation location = {
+ .filename = presumed_loc.getFilename(),
+ .line_number = presumed_loc.getLine(),
+ .column = presumed_loc.getColumn(),
+ .type = declaration_type,
+ .is_extern = is_extern,
+ .is_definition = is_definition,
+ .availability = availability,
+ };
+
+ // It's fine if the location is already there, we'll get an iterator to the existing element.
+ auto location_it = declaration_locations.begin();
+ bool inserted = false;
+ std::tie(location_it, inserted) = declaration_locations.insert(location);
+
+ // If we didn't insert, check to see if the availability attributes are identical.
+ if (!inserted) {
+ if (location_it->availability != availability) {
+ fprintf(stderr, "ERROR: availability attribute mismatch\n");
+ decl->dump();
+ abort();
+ }
+ }
+
+ return true;
+ }
+};
+
+void HeaderDatabase::parseAST(ASTUnit* ast) {
+ ASTContext& ctx = ast->getASTContext();
+ Visitor visitor(*this, ctx);
+ visitor.TraverseDecl(ctx.getTranslationUnitDecl());
+}
diff --git a/tools/versioner/src/DeclarationDatabase.h b/tools/versioner/src/DeclarationDatabase.h
new file mode 100644
index 0000000..057e416
--- /dev/null
+++ b/tools/versioner/src/DeclarationDatabase.h
@@ -0,0 +1,211 @@
+/*
+ * Copyright 2016 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <iostream>
+#include <map>
+#include <set>
+#include <string>
+#include <vector>
+
+#include <llvm/ADT/StringRef.h>
+
+#include "Utils.h"
+
+enum class DeclarationType {
+ function,
+ variable,
+ inconsistent,
+};
+
+static const char* declarationTypeName(DeclarationType type) {
+ switch (type) {
+ case DeclarationType::function:
+ return "function";
+ case DeclarationType::variable:
+ return "variable";
+ case DeclarationType::inconsistent:
+ return "inconsistent";
+ }
+}
+
+struct CompilationType {
+ std::string arch;
+ int api_level;
+
+ private:
+ auto tie() const {
+ return std::tie(arch, api_level);
+ }
+
+ public:
+ bool operator<(const CompilationType& other) const {
+ return tie() < other.tie();
+ }
+
+ bool operator==(const CompilationType& other) const {
+ return tie() == other.tie();
+ }
+
+ std::string describe() const {
+ return arch + "-" + std::to_string(api_level);
+ }
+};
+
+struct DeclarationAvailability {
+ int introduced = 0;
+ int deprecated = 0;
+ int obsoleted = 0;
+
+ void dump(std::ostream& out = std::cout) const {
+ bool need_comma = false;
+ auto comma = [&out, &need_comma]() {
+ if (!need_comma) {
+ need_comma = true;
+ return;
+ }
+ out << ", ";
+ };
+
+ if (introduced != 0) {
+ comma();
+ out << "introduced = " << introduced;
+ }
+ if (deprecated != 0) {
+ comma();
+ out << "deprecated = " << deprecated;
+ }
+ if (obsoleted != 0) {
+ comma();
+ out << "obsoleted = " << obsoleted;
+ }
+ }
+
+ bool empty() const {
+ return !(introduced || deprecated || obsoleted);
+ }
+
+ auto tie() const {
+ return std::tie(introduced, deprecated, obsoleted);
+ }
+
+ bool operator==(const DeclarationAvailability& rhs) const {
+ return this->tie() == rhs.tie();
+ }
+
+ bool operator!=(const DeclarationAvailability& rhs) const {
+ return !(*this == rhs);
+ }
+
+ std::string describe() const {
+ return std::string("[") + std::to_string(introduced) + "," + std::to_string(deprecated) + "," +
+ std::to_string(obsoleted) + "]";
+ }
+};
+
+struct DeclarationLocation {
+ std::string filename;
+ unsigned line_number;
+ unsigned column;
+ DeclarationType type;
+ bool is_extern;
+ bool is_definition;
+ DeclarationAvailability availability;
+
+ auto tie() const {
+ return std::tie(filename, line_number, column, type, is_extern, is_definition);
+ }
+
+ bool operator<(const DeclarationLocation& other) const {
+ return tie() < other.tie();
+ }
+
+ bool operator==(const DeclarationLocation& other) const {
+ return tie() == other.tie();
+ }
+};
+
+struct Declaration {
+ std::string name;
+ std::set<DeclarationLocation> locations;
+
+ bool hasDefinition() const {
+ for (const auto& location : locations) {
+ if (location.is_definition) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ DeclarationType type() const {
+ DeclarationType result = locations.begin()->type;
+ for (const DeclarationLocation& location : locations) {
+ if (location.type != result) {
+ result = DeclarationType::inconsistent;
+ }
+ }
+ return result;
+ }
+
+ void dump(const std::string& base_path = "", std::ostream& out = std::cout) const {
+ out << " " << name << " declared in " << locations.size() << " locations:\n";
+ for (const DeclarationLocation& location : locations) {
+ const char* var_type = declarationTypeName(location.type);
+ const char* declaration_type = location.is_definition ? "definition" : "declaration";
+ const char* linkage = location.is_extern ? "extern" : "static";
+
+ std::string filename;
+ if (llvm::StringRef(location.filename).startswith(base_path)) {
+ filename = location.filename.substr(base_path.size());
+ } else {
+ filename = location.filename;
+ }
+
+ out << " " << linkage << " " << var_type << " " << declaration_type << " @ "
+ << filename << ":" << location.line_number << ":" << location.column;
+
+ if (!location.availability.empty()) {
+ out << "\t[";
+ location.availability.dump(out);
+ out << "]";
+ } else {
+ out << "\t[no availability]";
+ }
+
+ out << "\n";
+ }
+ }
+};
+
+namespace clang {
+class ASTUnit;
+}
+
+class HeaderDatabase {
+ public:
+ std::map<std::string, Declaration> declarations;
+
+ void parseAST(clang::ASTUnit* ast);
+
+ void dump(const std::string& base_path = "", std::ostream& out = std::cout) const {
+ out << "HeaderDatabase contains " << declarations.size() << " declarations:\n";
+ for (const auto& pair : declarations) {
+ pair.second.dump(base_path, out);
+ }
+ }
+};
diff --git a/tools/versioner/src/SymbolDatabase.cpp b/tools/versioner/src/SymbolDatabase.cpp
new file mode 100644
index 0000000..a35f045
--- /dev/null
+++ b/tools/versioner/src/SymbolDatabase.cpp
@@ -0,0 +1,155 @@
+/*
+ * Copyright 2016 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "SymbolDatabase.h"
+
+#include <err.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <fstream>
+#include <streambuf>
+#include <string>
+#include <unordered_set>
+
+#include <llvm/ADT/SmallVector.h>
+#include <llvm/ADT/StringRef.h>
+#include <llvm/Object/Binary.h>
+#include <llvm/Object/ELFObjectFile.h>
+
+#include "versioner.h"
+
+using namespace llvm;
+using namespace llvm::object;
+
+std::unordered_set<std::string> getSymbols(const std::string& filename) {
+ std::unordered_set<std::string> result;
+ auto binary = createBinary(filename);
+ if (std::error_code ec = binary.getError()) {
+ errx(1, "failed to open library at %s: %s\n", filename.c_str(), ec.message().c_str());
+ }
+
+ ELFObjectFileBase* elf = dyn_cast_or_null<ELFObjectFileBase>(binary.get().getBinary());
+ if (!elf) {
+ errx(1, "failed to parse %s as ELF", filename.c_str());
+ }
+
+ for (const ELFSymbolRef symbol : elf->getDynamicSymbolIterators()) {
+ ErrorOr<StringRef> symbol_name = symbol.getName();
+
+ if (std::error_code ec = binary.getError()) {
+ errx(1, "failed to get symbol name for symbol in %s: %s", filename.c_str(),
+ ec.message().c_str());
+ }
+
+ result.insert(symbol_name.get().str());
+ }
+
+ return result;
+}
+
+// The NDK platforms are built by copying the platform directories on top of
+// each other to build each successive API version. Thus, we need to walk
+// backwards to find each desired file.
+static std::string readPlatformFile(const CompilationType& type, llvm::StringRef platform_dir,
+ const std::string& filename, bool required) {
+ int api_level = type.api_level;
+ std::ifstream stream;
+ while (api_level >= arch_min_api[type.arch]) {
+ if (supported_levels.count(api_level) == 0) {
+ --api_level;
+ continue;
+ }
+
+ std::string path = std::string(platform_dir) + "/android-" + std::to_string(api_level) +
+ "/arch-" + type.arch + "/symbols/" + filename;
+
+ stream = std::ifstream(path);
+ if (stream) {
+ return std::string(std::istreambuf_iterator<char>(stream), std::istreambuf_iterator<char>());
+ }
+
+ --api_level;
+ }
+
+ if (required) {
+ errx(1, "failed to find platform file '%s' for %s", filename.c_str(), type.describe().c_str());
+ }
+
+ return std::string();
+}
+
+static std::map<std::string, NdkSymbolType> parsePlatform(const CompilationType& type,
+ const std::string& platform_dir) {
+ std::map<std::string, NdkSymbolType> result;
+ std::map<std::string, bool /*required*/> wanted_files = {
+ { "libc.so.functions.txt", true },
+ { "libc.so.variables.txt", false },
+ { "libdl.so.functions.txt", false },
+ { "libm.so.functions.txt", false },
+ { "libm.so.variables.txt", false },
+ };
+
+ for (const auto& pair : wanted_files) {
+ llvm::StringRef file = pair.first;
+ bool required = pair.second;
+ NdkSymbolType symbol_type;
+ if (file.endswith(".functions.txt")) {
+ symbol_type = NdkSymbolType::function;
+ } else if (file.endswith(".variables.txt")) {
+ symbol_type = NdkSymbolType::variable;
+ } else {
+ errx(1, "internal error: unexpected platform filename '%s'\n", file.str().c_str());
+ }
+
+ std::string platform_file = readPlatformFile(type, platform_dir, file, required);
+ if (platform_file.empty()) {
+ continue;
+ }
+
+ llvm::SmallVector<llvm::StringRef, 0> symbols;
+ llvm::StringRef(platform_file).split(symbols, "\n");
+
+ for (llvm::StringRef symbol_name : symbols) {
+ if (symbol_name.empty()) {
+ continue;
+ }
+
+ if (result.count(symbol_name) != 0) {
+ if (verbose) {
+ printf("duplicated symbol '%s' in '%s'\n", symbol_name.str().c_str(), file.str().c_str());
+ }
+ }
+
+ result[symbol_name] = symbol_type;
+ }
+ }
+
+ return result;
+}
+
+NdkSymbolDatabase parsePlatforms(const std::set<CompilationType>& types,
+ const std::string& platform_dir) {
+ std::map<std::string, std::map<CompilationType, NdkSymbolType>> result;
+ for (const CompilationType& type : types) {
+ std::map<std::string, NdkSymbolType> symbols = parsePlatform(type, platform_dir);
+ for (const auto& it : symbols) {
+ result[it.first][type] = it.second;
+ }
+ }
+
+ return result;
+}
diff --git a/tools/versioner/src/SymbolDatabase.h b/tools/versioner/src/SymbolDatabase.h
new file mode 100644
index 0000000..3f41b98
--- /dev/null
+++ b/tools/versioner/src/SymbolDatabase.h
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2016 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <map>
+#include <set>
+#include <string>
+#include <unordered_set>
+
+#include "DeclarationDatabase.h"
+
+using LibrarySymbolDatabase = std::unordered_set<std::string>;
+std::unordered_set<std::string> getSymbols(const std::string& filename);
+
+enum class NdkSymbolType {
+ function,
+ variable,
+};
+
+using NdkSymbolDatabase = std::map<std::string, std::map<CompilationType, NdkSymbolType>>;
+NdkSymbolDatabase parsePlatforms(const std::set<CompilationType>& types,
+ const std::string& platform_dir);
diff --git a/tools/versioner/src/Utils.cpp b/tools/versioner/src/Utils.cpp
new file mode 100644
index 0000000..92cc9de
--- /dev/null
+++ b/tools/versioner/src/Utils.cpp
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2016 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "Utils.h"
+
+#include <err.h>
+#include <fts.h>
+#include <string.h>
+#include <unistd.h>
+
+#include <string>
+#include <vector>
+
+std::string getWorkingDir() {
+ char buf[PATH_MAX];
+ if (!getcwd(buf, sizeof(buf))) {
+ err(1, "getcwd failed");
+ }
+ return buf;
+}
+
+std::vector<std::string> collectFiles(const std::string& directory) {
+ std::vector<std::string> files;
+
+ char* dir_argv[2] = { const_cast<char*>(directory.c_str()), nullptr };
+ FTS* fts = fts_open(dir_argv, FTS_LOGICAL | FTS_NOCHDIR, nullptr);
+
+ if (!fts) {
+ err(1, "failed to open directory '%s'", directory.c_str());
+ }
+
+ FTSENT* ent;
+ while ((ent = fts_read(fts))) {
+ if (ent->fts_info & (FTS_D | FTS_DP)) {
+ continue;
+ }
+
+ files.push_back(ent->fts_path);
+ }
+
+ fts_close(fts);
+ return files;
+}
diff --git a/tools/versioner/src/Utils.h b/tools/versioner/src/Utils.h
new file mode 100644
index 0000000..f4845b8
--- /dev/null
+++ b/tools/versioner/src/Utils.h
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2016 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+std::string getWorkingDir();
+std::vector<std::string> collectFiles(const std::string& directory);
+
+namespace std {
+static __attribute__((unused)) std::string to_string(const char* c) {
+ return c;
+}
+
+static __attribute__((unused)) const std::string& to_string(const std::string& str) {
+ return str;
+}
+}
+
+template <typename Collection>
+static std::string Join(Collection c, const std::string& delimiter = ", ") {
+ std::string result;
+ for (const auto& item : c) {
+ result.append(std::to_string(item));
+ result.append(delimiter);
+ }
+ if (!result.empty()) {
+ result.resize(result.length() - delimiter.length());
+ }
+ return result;
+}
diff --git a/tools/versioner/src/versioner.cpp b/tools/versioner/src/versioner.cpp
new file mode 100644
index 0000000..1612ed0
--- /dev/null
+++ b/tools/versioner/src/versioner.cpp
@@ -0,0 +1,601 @@
+/*
+ * Copyright 2016 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <dirent.h>
+#include <err.h>
+#include <limits.h>
+#include <stdio.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include <atomic>
+#include <iostream>
+#include <map>
+#include <memory>
+#include <set>
+#include <sstream>
+#include <string>
+#include <thread>
+#include <unordered_map>
+#include <vector>
+
+#include <clang/Frontend/TextDiagnosticPrinter.h>
+#include <clang/Tooling/Tooling.h>
+#include <llvm/ADT/StringRef.h>
+
+#include "DeclarationDatabase.h"
+#include "SymbolDatabase.h"
+#include "Utils.h"
+#include "versioner.h"
+
+using namespace std::string_literals;
+using namespace clang;
+using namespace clang::tooling;
+
+bool verbose;
+
+class HeaderCompilationDatabase : public CompilationDatabase {
+ CompilationType type;
+ std::string cwd;
+ std::vector<std::string> headers;
+ std::vector<std::string> include_dirs;
+
+ public:
+ HeaderCompilationDatabase(CompilationType type, std::string cwd, std::vector<std::string> headers,
+ std::vector<std::string> include_dirs)
+ : type(type),
+ cwd(std::move(cwd)),
+ headers(std::move(headers)),
+ include_dirs(std::move(include_dirs)) {
+ }
+
+ CompileCommand generateCompileCommand(const std::string& filename) const {
+ std::vector<std::string> command = { "clang-tool", filename, "-nostdlibinc" };
+ for (const auto& dir : include_dirs) {
+ command.push_back("-isystem");
+ command.push_back(dir);
+ }
+ command.push_back("-std=c11");
+ command.push_back("-DANDROID");
+ command.push_back("-D__ANDROID_API__="s + std::to_string(type.api_level));
+ command.push_back("-D_FORTIFY_SOURCE=2");
+ command.push_back("-D_GNU_SOURCE");
+ command.push_back("-Wno-unknown-attributes");
+ command.push_back("-target");
+ command.push_back(arch_targets[type.arch]);
+
+ return CompileCommand(cwd, filename, command);
+ }
+
+ std::vector<CompileCommand> getAllCompileCommands() const override {
+ std::vector<CompileCommand> commands;
+ for (const std::string& file : headers) {
+ commands.push_back(generateCompileCommand(file));
+ }
+ return commands;
+ }
+
+ std::vector<CompileCommand> getCompileCommands(StringRef file) const override {
+ std::vector<CompileCommand> commands;
+ commands.push_back(generateCompileCommand(file));
+ return commands;
+ }
+
+ std::vector<std::string> getAllFiles() const override {
+ return headers;
+ }
+};
+
+struct CompilationRequirements {
+ std::vector<std::string> headers;
+ std::vector<std::string> dependencies;
+};
+
+static CompilationRequirements collectRequirements(const std::string& arch,
+ const std::string& header_dir,
+ const std::string& dependency_dir) {
+ std::vector<std::string> headers = collectFiles(header_dir);
+
+ std::vector<std::string> dependencies = { header_dir };
+ if (!dependency_dir.empty()) {
+ auto collect_children = [&dependencies](const std::string& dir_path) {
+ DIR* dir = opendir(dir_path.c_str());
+ if (!dir) {
+ err(1, "failed to open dependency directory '%s'", dir_path.c_str());
+ }
+
+ struct dirent* dent;
+ while ((dent = readdir(dir))) {
+ if (dent->d_name[0] == '.') {
+ continue;
+ }
+
+ // TODO: Resolve symlinks.
+ std::string dependency = dir_path + "/" + dent->d_name;
+
+ struct stat st;
+ if (stat(dependency.c_str(), &st) != 0) {
+ err(1, "failed to stat dependency '%s'", dependency.c_str());
+ }
+
+ if (!S_ISDIR(st.st_mode)) {
+ errx(1, "'%s' is not a directory", dependency.c_str());
+ }
+
+ dependencies.push_back(dependency);
+ }
+
+ closedir(dir);
+ };
+
+ collect_children(dependency_dir + "/common");
+ collect_children(dependency_dir + "/" + arch);
+ }
+
+ auto new_end = std::remove_if(headers.begin(), headers.end(), [&arch](llvm::StringRef header) {
+ for (const auto& it : header_blacklist) {
+ if (it.second.find(arch) == it.second.end()) {
+ continue;
+ }
+
+ if (header.endswith("/" + it.first)) {
+ return true;
+ }
+ }
+ return false;
+ });
+
+ headers.erase(new_end, headers.end());
+
+ CompilationRequirements result = { .headers = headers, .dependencies = dependencies };
+ return result;
+}
+
+static std::set<CompilationType> generateCompilationTypes(
+ const std::set<std::string> selected_architectures, const std::set<int>& selected_levels) {
+ std::set<CompilationType> result;
+ for (const std::string& arch : selected_architectures) {
+ int min_api = arch_min_api[arch];
+ for (int api_level : selected_levels) {
+ if (api_level < min_api) {
+ continue;
+ }
+ CompilationType type = { .arch = arch, .api_level = api_level };
+ result.insert(type);
+ }
+ }
+ return result;
+}
+
+using DeclarationDatabase = std::map<std::string, std::map<CompilationType, Declaration>>;
+
+static DeclarationDatabase transposeHeaderDatabases(
+ const std::map<CompilationType, HeaderDatabase>& original) {
+ DeclarationDatabase result;
+ for (const auto& outer : original) {
+ const CompilationType& type = outer.first;
+ for (const auto& inner : outer.second.declarations) {
+ const std::string& symbol_name = inner.first;
+ result[symbol_name][type] = inner.second;
+ }
+ }
+ return result;
+}
+
+static DeclarationDatabase compileHeaders(const std::set<CompilationType>& types,
+ const std::string& header_dir,
+ const std::string& dependency_dir, bool* failed) {
+ constexpr size_t thread_count = 8;
+ size_t threads_created = 0;
+ std::mutex mutex;
+ std::vector<std::thread> threads(thread_count);
+
+ std::map<CompilationType, HeaderDatabase> header_databases;
+ std::unordered_map<std::string, CompilationRequirements> requirements;
+
+ std::string cwd = getWorkingDir();
+ bool errors = false;
+
+ for (const auto& type : types) {
+ if (requirements.count(type.arch) == 0) {
+ requirements[type.arch] = collectRequirements(type.arch, header_dir, dependency_dir);
+ }
+ }
+
+ for (const auto& type : types) {
+ size_t thread_id = threads_created++;
+ if (thread_id >= thread_count) {
+ thread_id = thread_id % thread_count;
+ threads[thread_id].join();
+ }
+
+ threads[thread_id] = std::thread(
+ [&](CompilationType type) {
+ const auto& req = requirements[type.arch];
+
+ HeaderDatabase database;
+ HeaderCompilationDatabase compilation_database(type, cwd, req.headers, req.dependencies);
+
+ ClangTool tool(compilation_database, req.headers);
+
+ clang::DiagnosticOptions diagnostic_options;
+ std::vector<std::unique_ptr<ASTUnit>> asts;
+ tool.buildASTs(asts);
+ for (const auto& ast : asts) {
+ clang::DiagnosticsEngine& diagnostics_engine = ast->getDiagnostics();
+ if (diagnostics_engine.getNumWarnings() || diagnostics_engine.hasErrorOccurred()) {
+ std::unique_lock<std::mutex> l(mutex);
+ errors = true;
+ printf("versioner: compilation failure for %s in %s\n", type.describe().c_str(),
+ ast->getOriginalSourceFileName().str().c_str());
+ }
+
+ database.parseAST(ast.get());
+ }
+
+ std::unique_lock<std::mutex> l(mutex);
+ header_databases[type] = database;
+ },
+ type);
+ }
+
+ if (threads_created < thread_count) {
+ threads.resize(threads_created);
+ }
+
+ for (auto& thread : threads) {
+ thread.join();
+ }
+
+ if (errors) {
+ printf("versioner: compilation generated warnings or errors\n");
+ *failed = errors;
+ }
+
+ return transposeHeaderDatabases(header_databases);
+}
+
+static bool sanityCheck(const std::set<CompilationType>& types,
+ const DeclarationDatabase& database) {
+ bool error = false;
+ for (auto outer : database) {
+ const std::string& symbol_name = outer.first;
+ CompilationType last_type;
+ DeclarationAvailability last_availability;
+
+ // Rely on std::set being sorted to loop through the types by architecture.
+ for (const CompilationType& type : types) {
+ auto inner = outer.second.find(type);
+ if (inner == outer.second.end()) {
+ // TODO: Check for holes.
+ continue;
+ }
+
+ const Declaration& declaration = inner->second;
+ bool found_availability = false;
+ bool availability_mismatch = false;
+ DeclarationAvailability current_availability;
+
+ // Make sure that all of the availability declarations for this symbol match.
+ for (const DeclarationLocation& location : declaration.locations) {
+ if (!found_availability) {
+ found_availability = true;
+ current_availability = location.availability;
+ continue;
+ }
+
+ if (current_availability != location.availability) {
+ availability_mismatch = true;
+ error = true;
+ }
+ }
+
+ if (availability_mismatch) {
+ printf("%s: availability mismatch for %s\n", symbol_name.c_str(), type.describe().c_str());
+ declaration.dump(getWorkingDir() + "/");
+ }
+
+ if (type.arch != last_type.arch) {
+ last_type = type;
+ last_availability = current_availability;
+ continue;
+ }
+
+ // Make sure that availability declarations are consistent across API levels for a given arch.
+ if (last_availability != current_availability) {
+ error = true;
+ printf("%s: availability mismatch between %s and %s: %s before, %s after\n",
+ symbol_name.c_str(), last_type.describe().c_str(), type.describe().c_str(),
+ last_availability.describe().c_str(), current_availability.describe().c_str());
+ }
+
+ last_type = type;
+ }
+ }
+ return !error;
+}
+
+// Check that our symbol availability declarations match the actual NDK
+// platform symbol availability.
+static bool checkVersions(const std::set<CompilationType>& types,
+ const DeclarationDatabase& declaration_database,
+ const NdkSymbolDatabase& symbol_database) {
+ bool failed = false;
+
+ std::map<std::string, std::set<CompilationType>> arch_types;
+ for (const CompilationType& type : types) {
+ arch_types[type.arch].insert(type);
+ }
+
+ for (const auto& outer : declaration_database) {
+ const std::string& symbol_name = outer.first;
+ const auto& compilations = outer.second;
+
+ auto platform_availability_it = symbol_database.find(symbol_name);
+ if (platform_availability_it == symbol_database.end()) {
+ // This currently has lots of false positives (__INTRODUCED_IN_FUTURE, __errordecl, functions
+ // that come from crtbegin, etc.). Only print them with verbose, because of this.
+ if (verbose) {
+ printf("%s: not available in any platform\n", symbol_name.c_str());
+ }
+ continue;
+ }
+
+ const auto& platform_availability = platform_availability_it->second;
+ std::set<CompilationType> missing_symbol;
+ std::set<CompilationType> missing_decl;
+
+ for (const CompilationType& type : types) {
+ auto it = compilations.find(type);
+ if (it == compilations.end()) {
+ missing_decl.insert(type);
+ continue;
+ }
+
+ const Declaration& declaration = it->second;
+
+ // sanityCheck ensured that the availability declarations for a given arch match.
+ DeclarationAvailability availability = declaration.locations.begin()->availability;
+ int api_level = type.api_level;
+
+ int introduced = std::max(0, availability.introduced);
+ int obsoleted = availability.obsoleted == 0 ? INT_MAX : availability.obsoleted;
+ bool decl_available = api_level >= introduced && api_level < obsoleted;
+
+ auto symbol_availability_it = platform_availability.find(type);
+ bool symbol_available = symbol_availability_it != platform_availability.end();
+ if (decl_available) {
+ if (!symbol_available) {
+ // Make sure that either it exists in the platform, or an inline definition is visible.
+ if (!declaration.hasDefinition()) {
+ missing_symbol.insert(type);
+ continue;
+ }
+ } else {
+ // Make sure that symbols declared as functions/variables actually are.
+ switch (declaration.type()) {
+ case DeclarationType::inconsistent:
+ printf("%s: inconsistent declaration type\n", symbol_name.c_str());
+ declaration.dump();
+ exit(1);
+
+ case DeclarationType::variable:
+ if (symbol_availability_it->second != NdkSymbolType::variable) {
+ printf("%s: declared as variable, exists in platform as function\n",
+ symbol_name.c_str());
+ failed = true;
+ }
+ break;
+
+ case DeclarationType::function:
+ if (symbol_availability_it->second != NdkSymbolType::function) {
+ printf("%s: declared as function, exists in platform as variable\n",
+ symbol_name.c_str());
+ failed = true;
+ }
+ break;
+ }
+ }
+ } else {
+ // Make sure it's not available in the platform.
+ if (symbol_availability_it != platform_availability.end()) {
+ printf("%s: symbol should be unavailable in %s (declared with availability %s)\n",
+ symbol_name.c_str(), type.describe().c_str(), availability.describe().c_str());
+ failed = true;
+ }
+ }
+ }
+
+ // Allow declarations to be missing from an entire architecture.
+ for (const auto& arch_type : arch_types) {
+ const std::string& arch = arch_type.first;
+ bool found_all = true;
+ for (const auto& type : arch_type.second) {
+ if (missing_decl.find(type) == missing_decl.end()) {
+ found_all = false;
+ break;
+ }
+ }
+
+ if (!found_all) {
+ continue;
+ }
+
+ for (auto it = missing_decl.begin(); it != missing_decl.end();) {
+ if (it->arch == arch) {
+ it = missing_decl.erase(it);
+ } else {
+ ++it;
+ }
+ }
+ }
+
+ auto types_to_string = [](const std::set<CompilationType>& types) {
+ std::string result;
+ for (const CompilationType& type : types) {
+ result += type.describe();
+ result += ", ";
+ }
+ result.resize(result.length() - 2);
+ return result;
+ };
+
+ if (!missing_decl.empty()) {
+ printf("%s: declaration missing in %s\n", symbol_name.c_str(),
+ types_to_string(missing_decl).c_str());
+ failed = true;
+ }
+
+ if (!missing_symbol.empty()) {
+ printf("%s: declaration marked available but symbol missing in [%s]\n", symbol_name.c_str(),
+ types_to_string(missing_symbol).c_str());
+ failed = true;
+ }
+ }
+
+ return !failed;
+}
+
+static void usage() {
+ fprintf(stderr, "Usage: versioner [OPTION]... HEADER_PATH [DEPS_PATH]\n");
+ fprintf(stderr, "Version headers at HEADER_PATH, with DEPS_PATH/ARCH/* on the include path\n");
+ fprintf(stderr, "\n");
+ fprintf(stderr, "Target specification (defaults to all):\n");
+ fprintf(stderr, " -a API_LEVEL\tbuild with specified API level (can be repeated)\n");
+ fprintf(stderr, " \t\tvalid levels are %s\n", Join(supported_levels).c_str());
+ fprintf(stderr, " -r ARCH\tbuild with specified architecture (can be repeated)\n");
+ fprintf(stderr, " \t\tvalid architectures are %s\n", Join(supported_archs).c_str());
+ fprintf(stderr, "\n");
+ fprintf(stderr, "Validation:\n");
+ fprintf(stderr, " -p PATH\tcompare against NDK platform at PATH\n");
+ fprintf(stderr, " -d\t\tdump symbol availability in libraries\n");
+ fprintf(stderr, " -v\t\tenable verbose warnings\n");
+ exit(1);
+}
+
+int main(int argc, char** argv) {
+ std::string cwd = getWorkingDir() + "/";
+ bool default_args = true;
+ std::string platform_dir;
+ std::set<std::string> selected_architectures;
+ std::set<int> selected_levels;
+
+ int c;
+ while ((c = getopt(argc, argv, "a:r:p:n:duv")) != -1) {
+ default_args = false;
+ switch (c) {
+ case 'a': {
+ char* end;
+ int api_level = strtol(optarg, &end, 10);
+ if (end == optarg || strlen(end) > 0) {
+ usage();
+ }
+
+ if (supported_levels.count(api_level) == 0) {
+ errx(1, "unsupported API level %d", api_level);
+ }
+
+ selected_levels.insert(api_level);
+ break;
+ }
+
+ case 'r': {
+ if (supported_archs.count(optarg) == 0) {
+ errx(1, "unsupported architecture: %s", optarg);
+ }
+ selected_architectures.insert(optarg);
+ break;
+ }
+
+ case 'p': {
+ if (!platform_dir.empty()) {
+ usage();
+ }
+
+ platform_dir = optarg;
+
+ struct stat st;
+ if (stat(platform_dir.c_str(), &st) != 0) {
+ err(1, "failed to stat platform directory '%s'", platform_dir.c_str());
+ }
+ if (!S_ISDIR(st.st_mode)) {
+ errx(1, "'%s' is not a directory", optarg);
+ }
+ break;
+ }
+
+ case 'v':
+ verbose = true;
+ break;
+
+ default:
+ usage();
+ break;
+ }
+ }
+
+ if (argc - optind > 2 || optind >= argc) {
+ usage();
+ }
+
+ if (selected_levels.empty()) {
+ selected_levels = supported_levels;
+ }
+
+ if (selected_architectures.empty()) {
+ selected_architectures = supported_archs;
+ }
+
+ std::string dependencies = (argc - optind == 2) ? argv[optind + 1] : "";
+ const char* header_dir = argv[optind];
+
+ struct stat st;
+ if (stat(header_dir, &st) != 0) {
+ err(1, "failed to stat '%s'", header_dir);
+ } else if (!S_ISDIR(st.st_mode)) {
+ errx(1, "'%s' is not a directory", header_dir);
+ }
+
+ std::set<CompilationType> compilation_types;
+ DeclarationDatabase declaration_database;
+ NdkSymbolDatabase symbol_database;
+
+ compilation_types = generateCompilationTypes(selected_architectures, selected_levels);
+
+ // Do this before compiling so that we can early exit if the platforms don't match what we
+ // expect.
+ if (!platform_dir.empty()) {
+ symbol_database = parsePlatforms(compilation_types, platform_dir);
+ }
+
+ bool failed = false;
+ declaration_database = compileHeaders(compilation_types, header_dir, dependencies, &failed);
+
+ if (!sanityCheck(compilation_types, declaration_database)) {
+ printf("versioner: sanity check failed\n");
+ failed = true;
+ }
+
+ if (!platform_dir.empty()) {
+ if (!checkVersions(compilation_types, declaration_database, symbol_database)) {
+ printf("versioner: version check failed\n");
+ failed = true;
+ }
+ }
+
+ return failed;
+}
diff --git a/tools/versioner/src/versioner.h b/tools/versioner/src/versioner.h
new file mode 100644
index 0000000..95635bb
--- /dev/null
+++ b/tools/versioner/src/versioner.h
@@ -0,0 +1,57 @@
+/*
+ * Copyright 2016 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <map>
+#include <set>
+#include <string>
+#include <unordered_map>
+
+extern bool verbose;
+
+static const std::set<std::string> supported_archs = {
+ "arm", "arm64", "mips", "mips64", "x86", "x86_64",
+};
+
+static std::unordered_map<std::string, std::string> arch_targets = {
+ { "arm", "arm-linux-androideabi" },
+ { "arm64", "aarch64-linux-android" },
+ { "mips", "mipsel-linux-android" },
+ { "mips64", "mips64el-linux-android" },
+ { "x86", "i686-linux-android" },
+ { "x86_64", "x86_64-linux-android" },
+};
+
+static const std::set<int> supported_levels = { 9, 12, 13, 14, 15, 16, 17, 18, 19, 21, 23, 24 };
+
+// Non-const for the convenience of being able to index with operator[].
+static std::map<std::string, int> arch_min_api = {
+ { "arm", 9 },
+ { "arm64", 21 },
+ { "mips", 9 },
+ { "mips64", 21 },
+ { "x86", 9 },
+ { "x86_64", 21 },
+};
+
+static const std::unordered_map<std::string, std::set<std::string>> header_blacklist = {
+ // Internal header.
+ { "sys/_system_properties.h", supported_archs },
+
+ // time64.h #errors when included on LP64 archs.
+ { "time64.h", { "arm64", "mips64", "x86_64" } },
+};