tools/versioner/src/versioner.cpp

/*
 * Copyright (C) 2016 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <dirent.h>
#include <err.h>
#include <limits.h>
#include <stdio.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>

#include <atomic>
#include <functional>
#include <iostream>
#include <map>
#include <memory>
#include <set>
#include <sstream>
#include <string>
#include <thread>
#include <unordered_map>
#include <vector>

#include <clang/AST/ASTConsumer.h>
#include <clang/Basic/TargetInfo.h>
#include <clang/Driver/Compilation.h>
#include <clang/Driver/Driver.h>
#include <clang/Frontend/CompilerInstance.h>
#include <clang/Frontend/CompilerInvocation.h>
#include <clang/Frontend/FrontendAction.h>
#include <clang/Frontend/FrontendActions.h>
#include <clang/Frontend/TextDiagnosticPrinter.h>
#include <clang/Frontend/Utils.h>
#include <clang/FrontendTool/Utils.h>
#include <clang/Tooling/Tooling.h>
#include <llvm/ADT/StringRef.h>

#include <android-base/parseint.h>

#include "Arch.h"
#include "DeclarationDatabase.h"
#include "Preprocessor.h"
#include "SymbolDatabase.h"
#include "Utils.h"
#include "versioner.h"

using namespace std::string_literals;
using namespace clang;
using namespace tooling;

bool verbose;
static bool add_include;
static int max_thread_count = 48;

static std::vector<std::string> generateCompileCommand(CompilationType& type,
                                                       const std::string& filename,
                                                       const std::string& header_dir,
                                                       const std::vector<std::string>& include_dirs) {
  std::vector<std::string> cmd = { "versioner" };
  cmd.push_back("-std=c11");

  cmd.push_back("-Wall");
  cmd.push_back("-Wextra");
  cmd.push_back("-Werror");
  cmd.push_back("-Wundef");
  cmd.push_back("-Wno-unused-macros");
  cmd.push_back("-Wno-unused-function");
  cmd.push_back("-Wno-unused-variable");
  cmd.push_back("-Wno-unknown-attributes");
  cmd.push_back("-Wno-pragma-once-outside-header");

  cmd.push_back("-target");
  cmd.push_back(arch_targets[type.arch]);

  cmd.push_back("-DANDROID");
  cmd.push_back("-D__ANDROID_API__="s + std::to_string(type.api_level));
  cmd.push_back("-D_FORTIFY_SOURCE=2");
  cmd.push_back("-D_GNU_SOURCE");
  cmd.push_back("-D_FILE_OFFSET_BITS="s + std::to_string(type.file_offset_bits));

  cmd.push_back("-nostdinc");
  std::string header_path;
  if (add_include) {
    const char* top = getenv("ANDROID_BUILD_TOP");
    header_path = to_string(top) + "/bionic/libc/include/android/versioning.h";
    cmd.push_back("-include");
    cmd.push_back(header_path);
  }

  for (const auto& dir : include_dirs) {
    cmd.push_back("-isystem");
    cmd.push_back(dir);
  }

  cmd.push_back(filename);

  return cmd;
}

class VersionerASTConsumer : public clang::ASTConsumer {
 public:
  HeaderDatabase* header_database;
  CompilationType type;

  VersionerASTConsumer(HeaderDatabase* header_database, CompilationType type)
      : header_database(header_database), type(type) {
  }

  virtual void HandleTranslationUnit(ASTContext& ctx) override {
    header_database->parseAST(type, ctx);
  }
};

class VersionerASTAction : public clang::ASTFrontendAction {
 public:
  HeaderDatabase* header_database;
  CompilationType type;

  VersionerASTAction(HeaderDatabase* header_database, CompilationType type)
      : header_database(header_database), type(type) {
  }

  virtual std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance& Compiler,
                                                         llvm::StringRef InFile) override {
    return std::make_unique<VersionerASTConsumer>(header_database, type);
  }
};

static void compileHeader(HeaderDatabase* header_database, CompilationType type,
                          const std::string& filename, const std::string& header_dir,
                          const std::vector<std::string>& include_dirs) {
  DiagnosticOptions diagnostic_options;
  auto diagnostic_printer = new TextDiagnosticPrinter(llvm::errs(), &diagnostic_options);

  IntrusiveRefCntPtr<DiagnosticIDs> diagnostic_ids(new DiagnosticIDs());
  IntrusiveRefCntPtr<DiagnosticsEngine> diags(
      new DiagnosticsEngine(diagnostic_ids, &diagnostic_options, diagnostic_printer, false));
  driver::Driver Driver("versioner", llvm::sys::getDefaultTargetTriple(), *diags);

  std::vector<std::string> cmd = generateCompileCommand(type, filename, header_dir, include_dirs);
  llvm::SmallVector<const char*, 32> Args;

  for (const std::string& str : cmd) {
    Args.push_back(str.c_str());
  }

  std::unique_ptr<driver::Compilation> Compilation(Driver.BuildCompilation(Args));

  const driver::Command &Cmd = llvm::cast<driver::Command>(*Compilation->getJobs().begin());
  const driver::ArgStringList &CCArgs = Cmd.getArguments();

  auto invocation = std::make_unique<CompilerInvocation>();
  if (!CompilerInvocation::CreateFromArgs(
          *invocation.get(), const_cast<const char**>(CCArgs.data()),
          const_cast<const char**>(CCArgs.data()) + CCArgs.size(), *diags)) {
    errx(1, "failed to create CompilerInvocation");
  }

  clang::CompilerInstance Compiler;
  Compiler.setInvocation(invocation.release());
  Compiler.setDiagnostics(diags.get());

  VersionerASTAction versioner_action(header_database, type);
  if (!Compiler.ExecuteAction(versioner_action)) {
    errx(1, "compilation generated warnings or errors");
  }

  if (diags->getNumWarnings() || diags->hasErrorOccurred()) {
    errx(1, "compilation generated warnings or errors");
  }
}

struct CompilationRequirements {
  std::vector<std::string> headers;
  std::vector<std::string> dependencies;
};

static CompilationRequirements collectRequirements(const Arch& arch, const std::string& header_dir,
                                                   const std::string& dependency_dir) {
  std::vector<std::string> headers = collectFiles(header_dir);

  std::vector<std::string> dependencies = { header_dir };
  if (!dependency_dir.empty()) {
    auto collect_children = [&dependencies](const std::string& dir_path) {
      DIR* dir = opendir(dir_path.c_str());
      if (!dir) {
        err(1, "failed to open dependency directory '%s'", dir_path.c_str());
      }

      struct dirent* dent;
      while ((dent = readdir(dir))) {
        if (dent->d_name[0] == '.') {
          continue;
        }

        // TODO: Resolve symlinks.
        std::string dependency = dir_path + "/" + dent->d_name;

        struct stat st;
        if (stat(dependency.c_str(), &st) != 0) {
          err(1, "failed to stat dependency '%s'", dependency.c_str());
        }

        if (!S_ISDIR(st.st_mode)) {
          errx(1, "'%s' is not a directory", dependency.c_str());
        }

        dependencies.push_back(dependency);
      }

      closedir(dir);
    };

    collect_children(dependency_dir + "/common");
    collect_children(dependency_dir + "/" + to_string(arch));
  }

  auto new_end = std::remove_if(headers.begin(), headers.end(), [&arch](llvm::StringRef header) {
    for (const auto& it : header_blacklist) {
      if (it.second.find(arch) == it.second.end()) {
        continue;
      }

      if (header.endswith("/" + it.first)) {
        return true;
      }
    }
    return false;
  });

  headers.erase(new_end, headers.end());

  CompilationRequirements result = { .headers = headers, .dependencies = dependencies };
  return result;
}

static std::set<CompilationType> generateCompilationTypes(const std::set<Arch> selected_architectures,
                                                          const std::set<int>& selected_levels) {
  std::set<CompilationType> result;
  for (const auto& arch : selected_architectures) {
    int min_api = arch_min_api[arch];
    for (int api_level : selected_levels) {
      if (api_level < min_api) {
        continue;
      }

      for (int file_offset_bits : { 32, 64 }) {
        CompilationType type = {
          .arch = arch, .api_level = api_level, .file_offset_bits = file_offset_bits
        };
        result.insert(type);
      }
    }
  }
  return result;
}

struct Job {
  Job(CompilationType type, const std::string& header, const std::vector<std::string>& dependencies)
      : type(type), header(header), dependencies(dependencies) {
  }
  CompilationType type;
  const std::string& header;
  const std::vector<std::string>& dependencies;
};

static std::unique_ptr<HeaderDatabase> compileHeaders(const std::set<CompilationType>& types,
                                                      const std::string& header_dir,
                                                      const std::string& dependency_dir) {
  if (types.empty()) {
    errx(1, "compileHeaders received no CompilationTypes");
  }

  size_t thread_count = max_thread_count;
  std::vector<std::thread> threads;
  std::vector<Job> jobs;

  std::map<CompilationType, HeaderDatabase> header_databases;
  std::unordered_map<Arch, CompilationRequirements> requirements;

  std::string cwd = getWorkingDir();

  auto result = std::make_unique<HeaderDatabase>();
  auto spawn_threads = [&]() {
    thread_count = std::min(thread_count, jobs.size());
    for (size_t i = 0; i < thread_count; ++i) {
      threads.emplace_back([&jobs, &result, &header_dir, thread_count, i]() {
        size_t index = i;
        while (index < jobs.size()) {
          const auto& job = jobs[index];
          compileHeader(result.get(), job.type, job.header, header_dir, job.dependencies);
          index += thread_count;
        }
      });
    }
  };
  auto reap_threads = [&]() {
    for (auto& thread : threads) {
      thread.join();
    }
    threads.clear();
  };

  for (const auto& type : types) {
    if (requirements.count(type.arch) == 0) {
      requirements[type.arch] = collectRequirements(type.arch, header_dir, dependency_dir);
    }
  }

  for (CompilationType type : types) {
    CompilationRequirements& req = requirements[type.arch];
    for (const std::string& header : req.headers) {
      jobs.emplace_back(type, header, req.dependencies);
    }
  }

  spawn_threads();
  reap_threads();

  return result;
}

// Perform a sanity check on a symbol's declarations, enforcing the following invariants:
//   1. At most one inline definition of the function exists.
//   2. All of the availability declarations for a symbol are compatible.
//      If a function is declared as an inline before a certain version, the inline definition
//      should have no version tag.
//   3. Each availability type must only be present globally or on a per-arch basis.
//      (e.g. __INTRODUCED_IN_ARM(9) __INTRODUCED_IN_X86(10) __DEPRECATED_IN(11) is fine,
//      but not __INTRODUCED_IN(9) __INTRODUCED_IN_X86(10))
static bool checkSymbol(const Symbol& symbol) {
  std::string cwd = getWorkingDir() + "/";

  const Declaration* inline_definition = nullptr;
  for (const auto& decl_it : symbol.declarations) {
    const Declaration* decl = &decl_it.second;
    if (decl->is_definition) {
      if (inline_definition) {
        fprintf(stderr, "versioner: multiple definitions of symbol %s\n", symbol.name.c_str());
        symbol.dump(cwd);
        inline_definition->dump(cwd);
        return false;
      }

      inline_definition = decl;
    }

    DeclarationAvailability availability;
    if (!decl->calculateAvailability(&availability)) {
      fprintf(stderr, "versioner: failed to calculate availability for declaration:\n");
      decl->dump(cwd, stderr, 2);
      return false;
    }

    if (decl->is_definition && !availability.empty()) {
      fprintf(stderr, "versioner: inline definition has non-empty versioning information:\n");
      decl->dump(cwd, stderr, 2);
      return false;
    }
  }

  DeclarationAvailability availability;
  if (!symbol.calculateAvailability(&availability)) {
    fprintf(stderr, "versioner: inconsistent availability for symbol '%s'\n", symbol.name.c_str());
    symbol.dump(cwd);
    return false;
  }

  // TODO: Check invariant #3.
  return true;
}

static bool sanityCheck(const HeaderDatabase* database) {
  bool error = false;
  std::string cwd = getWorkingDir() + "/";

  for (const auto& symbol_it : database->symbols) {
    if (!checkSymbol(symbol_it.second)) {
      error = true;
    }
  }
  return !error;
}

// Check that our symbol availability declarations match the actual NDK
// platform symbol availability.
static bool checkVersions(const std::set<CompilationType>& types,
                          const HeaderDatabase* header_database,
                          const NdkSymbolDatabase& symbol_database) {
  std::string cwd = getWorkingDir() + "/";
  bool failed = false;

  std::map<Arch, std::set<CompilationType>> arch_types;
  for (const CompilationType& type : types) {
    arch_types[type.arch].insert(type);
  }

  std::set<std::string> completely_unavailable;
  std::map<std::string, std::set<CompilationType>> missing_availability;
  std::map<std::string, std::set<CompilationType>> extra_availability;

  for (const auto& symbol_it : header_database->symbols) {
    const auto& symbol_name = symbol_it.first;
    DeclarationAvailability symbol_availability;

    if (!symbol_it.second.calculateAvailability(&symbol_availability)) {
      errx(1, "failed to calculate symbol availability");
    }

    const auto platform_availability_it = symbol_database.find(symbol_name);
    if (platform_availability_it == symbol_database.end()) {
      completely_unavailable.insert(symbol_name);
      continue;
    }

    const auto& platform_availability = platform_availability_it->second;

    for (const CompilationType& type : types) {
      bool should_be_available = true;
      const auto& global_availability = symbol_availability.global_availability;
      const auto& arch_availability = symbol_availability.arch_availability[type.arch];
      if (global_availability.introduced != 0 && global_availability.introduced > type.api_level) {
        should_be_available = false;
      }

      if (arch_availability.introduced != 0 && arch_availability.introduced > type.api_level) {
        should_be_available = false;
      }

      if (global_availability.obsoleted != 0 && global_availability.obsoleted <= type.api_level) {
        should_be_available = false;
      }

      if (arch_availability.obsoleted != 0 && arch_availability.obsoleted <= type.api_level) {
        should_be_available = false;
      }

      if (arch_availability.future) {
        continue;
      }

      // The function declaration might be (validly) missing for the given CompilationType.
      if (!symbol_it.second.hasDeclaration(type)) {
        should_be_available = false;
      }

      bool is_available = platform_availability.count(type);

      if (should_be_available != is_available) {
        if (is_available) {
          extra_availability[symbol_name].insert(type);
        } else {
          missing_availability[symbol_name].insert(type);
        }
      }
    }
  }

  for (const auto& it : symbol_database) {
    const std::string& symbol_name = it.first;

    bool symbol_error = false;
    auto missing_it = missing_availability.find(symbol_name);
    if (missing_it != missing_availability.end()) {
      printf("%s: declaration marked available but symbol missing in [%s]\n", symbol_name.c_str(),
             Join(missing_it->second, ", ").c_str());
      symbol_error = true;
      failed = true;
    }

    if (verbose) {
      auto extra_it = extra_availability.find(symbol_name);
      if (extra_it != extra_availability.end()) {
        printf("%s: declaration marked unavailable but symbol available in [%s]\n",
               symbol_name.c_str(), Join(extra_it->second, ", ").c_str());
        symbol_error = true;
        failed = true;
      }
    }

    if (symbol_error) {
      auto symbol_it = header_database->symbols.find(symbol_name);
      if (symbol_it == header_database->symbols.end()) {
        errx(1, "failed to find symbol in header database");
      }
      symbol_it->second.dump(cwd);
    }
  }

  // TODO: Verify that function/variable declarations are actually function/variable symbols.
  return !failed;
}

static void usage(bool help = false) {
  fprintf(stderr, "Usage: versioner [OPTION]... [HEADER_PATH] [DEPS_PATH]\n");
  if (!help) {
    printf("Try 'versioner -h' for more information.\n");
    exit(1);
  } else {
    fprintf(stderr, "Version headers at HEADER_PATH, with DEPS_PATH/ARCH/* on the include path\n");
    fprintf(stderr, "Autodetects paths if HEADER_PATH and DEPS_PATH are not specified\n");
    fprintf(stderr, "\n");
    fprintf(stderr, "Target specification (defaults to all):\n");
    fprintf(stderr, "  -a API_LEVEL\tbuild with specified API level (can be repeated)\n");
    fprintf(stderr, "    \t\tvalid levels are %s\n", Join(supported_levels).c_str());
    fprintf(stderr, "  -r ARCH\tbuild with specified architecture (can be repeated)\n");
    fprintf(stderr, "    \t\tvalid architectures are %s\n", Join(supported_archs).c_str());
    fprintf(stderr, "\n");
    fprintf(stderr, "Validation:\n");
    fprintf(stderr, "  -p PATH\tcompare against NDK platform at PATH\n");
    fprintf(stderr, "  -v\t\tenable verbose warnings\n");
    fprintf(stderr, "\n");
    fprintf(stderr, "Preprocessing:\n");
    fprintf(stderr, "  -o PATH\tpreprocess header files and emit them at PATH\n");
    fprintf(stderr, "  -f\tpreprocess header files even if validation fails\n");
    fprintf(stderr, "\n");
    fprintf(stderr, "Miscellaneous:\n");
    fprintf(stderr, "  -d\t\tdump function availability\n");
    fprintf(stderr, "  -j THREADS\tmaximum number of threads to use\n");
    fprintf(stderr, "  -h\t\tdisplay this message\n");
    exit(0);
  }
}

int main(int argc, char** argv) {
  std::string cwd = getWorkingDir() + "/";
  bool default_args = true;
  std::string platform_dir;
  std::set<Arch> selected_architectures;
  std::set<int> selected_levels;
  std::string preprocessor_output_path;
  bool force = false;
  bool dump = false;

  int c;
  while ((c = getopt(argc, argv, "a:r:p:vo:fdj:hi")) != -1) {
    default_args = false;
    switch (c) {
      case 'a': {
        char* end;
        int api_level = strtol(optarg, &end, 10);
        if (end == optarg || strlen(end) > 0) {
          usage();
        }

        if (supported_levels.count(api_level) == 0) {
          errx(1, "unsupported API level %d", api_level);
        }

        selected_levels.insert(api_level);
        break;
      }

      case 'r': {
        Arch arch = arch_from_string(optarg);
        selected_architectures.insert(arch);
        break;
      }

      case 'p': {
        if (!platform_dir.empty()) {
          usage();
        }

        platform_dir = optarg;

        if (platform_dir.empty()) {
          usage();
        }

        struct stat st;
        if (stat(platform_dir.c_str(), &st) != 0) {
          err(1, "failed to stat platform directory '%s'", platform_dir.c_str());
        }
        if (!S_ISDIR(st.st_mode)) {
          errx(1, "'%s' is not a directory", optarg);
        }
        break;
      }

      case 'v':
        verbose = true;
        break;

      case 'o':
        if (!preprocessor_output_path.empty()) {
          usage();
        }
        preprocessor_output_path = optarg;
        if (preprocessor_output_path.empty()) {
          usage();
        }
        break;

      case 'f':
        force = true;
        break;

      case 'd':
        dump = true;
        break;

      case 'j':
        if (!android::base::ParseInt<int>(optarg, &max_thread_count, 1)) {
          usage();
        }
        break;

      case 'h':
        usage(true);
        break;

      case 'i':
        // Secret option for tests to -include <android/versioning.h>.
        add_include = true;
        break;

      default:
        usage();
        break;
    }
  }

  if (argc - optind > 2 || optind > argc) {
    usage();
  }

  std::string header_dir;
  std::string dependency_dir;

  const char* top = getenv("ANDROID_BUILD_TOP");
  if (!top && (optind == argc || add_include)) {
    fprintf(stderr, "versioner: failed to autodetect bionic paths. Is ANDROID_BUILD_TOP set?\n");
    usage();
  }

  if (optind == argc) {
    // Neither HEADER_PATH nor DEPS_PATH were specified, so try to figure them out.
    std::string versioner_dir = to_string(top) + "/bionic/tools/versioner";
    header_dir = versioner_dir + "/current";
    dependency_dir = versioner_dir + "/dependencies";
    if (platform_dir.empty()) {
      platform_dir = versioner_dir + "/platforms";
    }
  } else {
    // Intentional leak.
    header_dir = realpath(argv[optind], nullptr);

    if (argc - optind == 2) {
      dependency_dir = argv[optind + 1];
    }
  }

  if (selected_levels.empty()) {
    selected_levels = supported_levels;
  }

  if (selected_architectures.empty()) {
    selected_architectures = supported_archs;
  }


  struct stat st;
  if (stat(header_dir.c_str(), &st) != 0) {
    err(1, "failed to stat '%s'", header_dir.c_str());
  } else if (!S_ISDIR(st.st_mode)) {
    errx(1, "'%s' is not a directory", header_dir.c_str());
  }

  std::set<CompilationType> compilation_types;
  NdkSymbolDatabase symbol_database;

  compilation_types = generateCompilationTypes(selected_architectures, selected_levels);

  // Do this before compiling so that we can early exit if the platforms don't match what we
  // expect.
  if (!platform_dir.empty()) {
    symbol_database = parsePlatforms(compilation_types, platform_dir);
  }

  std::unique_ptr<HeaderDatabase> declaration_database =
      compileHeaders(compilation_types, header_dir, dependency_dir);

  bool failed = false;
  if (dump) {
    declaration_database->dump(header_dir + "/");
  } else {
    if (!sanityCheck(declaration_database.get())) {
      printf("versioner: sanity check failed\n");
      failed = true;
    }

    if (!platform_dir.empty()) {
      if (!checkVersions(compilation_types, declaration_database.get(), symbol_database)) {
        printf("versioner: version check failed\n");
        failed = true;
      }
    }
  }

  if (!preprocessor_output_path.empty() && (force || !failed)) {
    failed = !preprocessHeaders(preprocessor_output_path, header_dir, declaration_database.get());
  }
  return failed;
}