orchestrator/demo/collect_metadata.py

#!/usr/bin/env python3
# Copyright (C) 2021 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import copy
import json
import logging
import os
import sys
import yaml
from collections import defaultdict
from typing import (
  List,
  Set,
)

import utils

# SKIP_COMPONENT_SEARCH = (
#    'tools',
# )
COMPONENT_METADATA_DIR = '.repo'
COMPONENT_METADATA_FILE = 'treeinfo.yaml'
GENERATED_METADATA_FILE = 'metadata.json'
COMBINED_METADATA_FILENAME = 'multitree_meta.json'


class Dep(object):
  def __init__(self, name, component, deps_type):
    self.name = name
    self.component = component
    self.type = deps_type
    self.out_paths = list()


class ExportedDep(Dep):
  def __init__(self, name, component, deps_type):
    super().__init__(name, component, deps_type)

  def setOutputPaths(self, output_paths: list):
    self.out_paths = output_paths


class ImportedDep(Dep):
  required_type_map = {
    # import type: (required type, get imported module list)
    utils.META_FILEGROUP: (utils.META_MODULES, True),
  }

  def __init__(self, name, component, deps_type, import_map):
    super().__init__(name, component, deps_type)
    self.exported_deps: Set[ExportedDep] = set()
    self.imported_modules: List[str] = list()
    self.required_type = deps_type
    get_imported_module = False
    if deps_type in ImportedDep.required_type_map:
      self.required_type, get_imported_module = ImportedDep.required_type_map[deps_type]
    if get_imported_module:
      self.imported_modules = import_map[name]
    else:
      self.imported_modules.append(name)

  def verify_and_add(self, exported: ExportedDep):
    if self.required_type != exported.type:
      raise RuntimeError(
          '{comp} components imports {module} for {imp_type} but it is exported as {exp_type}.'
          .format(comp=self.component, module=exported.name, imp_type=self.required_type, exp_type=exported.type))
    self.exported_deps.add(exported)
    self.out_paths.extend(exported.out_paths)
    # Remove duplicates. We may not use set() which is not JSON serializable
    self.out_paths = list(dict.fromkeys(self.out_paths))


class MetadataCollector(object):
  """Visit all component directories and collect the metadata from them.

Example of metadata:
==========
build_cmd: m    # build command for this component. 'm' if omitted
out_dir: out    # out dir of this component. 'out' if omitted
exports:
  libraries:
    - name: libopenjdkjvm
    - name: libopenjdkjvmd
      build_cmd: mma      # build command for libopenjdkjvmd if specified
      out_dir: out/soong  # out dir for libopenjdkjvmd if specified
    - name: libctstiagent
  APIs:
    - api1
    - api2
imports:
  libraries:
    - lib1
    - lib2
  APIs:
    - import_api1
    - import_api2
lunch_targets:
  - arm64
  - x86_64
"""

  def __init__(self, component_top, out_dir, meta_dir, meta_file, force_update=False):
    if not os.path.exists(out_dir):
      os.makedirs(out_dir)

    self.__component_top = component_top
    self.__out_dir = out_dir
    self.__metadata_path = os.path.join(meta_dir, meta_file)
    self.__combined_metadata_path = os.path.join(self.__out_dir,
                                                 COMBINED_METADATA_FILENAME)
    self.__force_update = force_update

    self.__metadata = dict()
    self.__map_exports = dict()
    self.__component_set = set()

  def collect(self):
    """ Read precomputed combined metadata from the json file.

    If any components have updated their metadata, update the metadata
    information and the json file.
    """
    timestamp = self.__restore_metadata()
    if timestamp and os.path.getmtime(__file__) > timestamp:
      logging.info('Update the metadata as the orchestrator has been changed')
      self.__force_update = True
    self.__collect_from_components(timestamp)

  def get_metadata(self):
    """ Returns collected metadata from all components"""
    if not self.__metadata:
      logging.warning('Metadata is empty')
    return copy.deepcopy(self.__metadata)

  def __collect_from_components(self, timestamp):
    """ Read metadata from all components

    If any components have newer metadata files or are removed, update the
    combined metadata.
    """
    metadata_updated = False
    for component in os.listdir(self.__component_top):
      # if component in SKIP_COMPONENT_SEARCH:
      #     continue
      if self.__read_component_metadata(timestamp, component):
        metadata_updated = True
      if self.__read_generated_metadata(timestamp, component):
        metadata_updated = True

    deleted_components = set()
    for meta in self.__metadata:
      if meta not in self.__component_set:
        logging.info('Component {} is removed'.format(meta))
        deleted_components.add(meta)
        metadata_updated = True
    for meta in deleted_components:
      del self.__metadata[meta]

    if metadata_updated:
      self.__update_dependencies()
      self.__store_metadata()
      logging.info('Metadata updated')

  def __read_component_metadata(self, timestamp, component):
    """ Search for the metadata file from a component.

    If the metadata is modified, read the file and update the metadata.
    """
    component_path = os.path.join(self.__component_top, component)
    metadata_file = os.path.join(component_path, self.__metadata_path)
    logging.info(
        'Reading a metadata file from {} component ...'.format(component))
    if not os.path.isfile(metadata_file):
      logging.warning('Metadata file {} not found!'.format(metadata_file))
      return False

    self.__component_set.add(component)
    if not self.__force_update and timestamp and timestamp > os.path.getmtime(metadata_file):
      logging.info('... yaml not changed. Skip')
      return False

    with open(metadata_file) as f:
      meta = yaml.load(f, Loader=yaml.SafeLoader)

    meta['path'] = component_path
    if utils.META_BUILDCMD not in meta:
      meta[utils.META_BUILDCMD] = utils.DEFAULT_BUILDCMD
    if utils.META_OUTDIR not in meta:
      meta[utils.META_OUTDIR] = utils.DEFAULT_OUTDIR

    if utils.META_IMPORTS not in meta:
      meta[utils.META_IMPORTS] = defaultdict(dict)
    if utils.META_EXPORTS not in meta:
      meta[utils.META_EXPORTS] = defaultdict(dict)

    self.__metadata[component] = meta
    return True

  def __read_generated_metadata(self, timestamp, component):
    """ Read a metadata gerated by 'update-meta' build command from the soong build system

    Soong generate the metadata that has the information of import/export module/files.
    Build orchestrator read the generated metadata to collect the dependency information.

    Generated metadata has the following format:
    {
      "Imported": {
        "FileGroups": {
          "<name_of_filegroup>": [
            "<exported_module_name>",
            ...
          ],
          ...
        }
      }
      "Exported": {
        "<exported_module_name>": [
          "<output_file_path>",
          ...
        ],
        ...
      }
    }
    """
    if component not in self.__component_set:
      # skip reading generated metadata if the component metadata file was missing
      return False
    component_out = os.path.join(self.__component_top, component, self.__metadata[component][utils.META_OUTDIR])
    generated_metadata_file = os.path.join(component_out, 'soong', 'multitree', GENERATED_METADATA_FILE)
    if not os.path.isfile(generated_metadata_file):
      logging.info('... Soong did not generated the metadata file. Skip')
      return False
    if not self.__force_update and timestamp and timestamp > os.path.getmtime(generated_metadata_file):
      logging.info('... Soong generated metadata not changed. Skip')
      return False

    with open(generated_metadata_file, 'r') as gen_meta_json:
      try:
        gen_metadata = json.load(gen_meta_json)
      except json.decoder.JSONDecodeError:
        logging.warning('JSONDecodeError!!!: skip reading the {} file'.format(
            generated_metadata_file))
        return False

    if utils.SOONG_IMPORTED in gen_metadata:
      imported = gen_metadata[utils.SOONG_IMPORTED]
      if utils.SOONG_IMPORTED_FILEGROUPS in imported:
        self.__metadata[component][utils.META_IMPORTS][utils.META_FILEGROUP] = imported[utils.SOONG_IMPORTED_FILEGROUPS]
    if utils.SOONG_EXPORTED in gen_metadata:
      self.__metadata[component][utils.META_EXPORTS][utils.META_MODULES] = gen_metadata[utils.SOONG_EXPORTED]

    return True

  def __update_export_map(self):
    """ Read metadata of all components and update the export map

    'libraries' and 'APIs' are special exproted types that are provided manually
    from the .yaml metadata files. These need to be replaced with the implementation
    in soong gerated metadata.
    The export type 'module' is generated from the soong build system from the modules
    with 'export: true' property. This export type includes a dictionary with module
    names as keys and their output files as values. These output files will be used as
    prebuilt sources when generating the imported modules.
    """
    self.__map_exports = dict()
    for comp in self.__metadata:
      if utils.META_EXPORTS not in self.__metadata[comp]:
        continue
      exports = self.__metadata[comp][utils.META_EXPORTS]

      for export_type in exports:
        for module in exports[export_type]:
          if export_type == utils.META_LIBS:
            name = module[utils.META_LIB_NAME]
          else:
            name = module

          if name in self.__map_exports:
            raise RuntimeError(
                'Exported libs conflict!!!: "{name}" in the {comp} component is already exported by the {prev} component.'
                .format(name=name, comp=comp, prev=self.__map_exports[name][utils.EXP_COMPONENT]))
          exported_deps = ExportedDep(name, comp, export_type)
          if export_type == utils.META_MODULES:
            exported_deps.setOutputPaths(exports[export_type][module])
          self.__map_exports[name] = exported_deps

  def __verify_and_add_dependencies(self, component):
    """ Search all imported items from the export_map.

    If any imported items are not provided by the other components, report
    an error.
    Otherwise, add the component dependency and update the exported information to the
    import maps.
    """
    def verify_and_add_dependencies(imported_dep: ImportedDep):
      for module in imported_dep.imported_modules:
        if module not in self.__map_exports:
          raise RuntimeError(
              'Imported item not found!!!: Imported module "{module}" in the {comp} component is not exported from any other components.'
              .format(module=module, comp=imported_dep.component))
        imported_dep.verify_and_add(self.__map_exports[module])

        deps = self.__metadata[component][utils.META_DEPS]
        exp_comp = self.__map_exports[module].component
        if exp_comp not in deps:
          deps[exp_comp] = defaultdict(defaultdict)
        deps[exp_comp][imported_dep.type][imported_dep.name] = imported_dep.out_paths

    self.__metadata[component][utils.META_DEPS] = defaultdict()
    imports = self.__metadata[component][utils.META_IMPORTS]
    for import_type in imports:
      for module in imports[import_type]:
        verify_and_add_dependencies(ImportedDep(module, component, import_type, imports[import_type]))

  def __check_imports(self):
    """ Search the export map to find the component to import libraries or APIs.

    Update the 'deps' field that includes the dependent components.
    """
    for component in self.__metadata:
      self.__verify_and_add_dependencies(component)
      if utils.META_DEPS in self.__metadata[component]:
        logging.debug('{comp} depends on {list} components'.format(
            comp=component, list=self.__metadata[component][utils.META_DEPS]))

  def __update_dependencies(self):
    """ Generate a dependency graph for the components

    Update __map_exports and the dependency graph with the maps.
    """
    self.__update_export_map()
    self.__check_imports()

  def __store_metadata(self):
    """ Store the __metadata dictionary as json format"""
    with open(self.__combined_metadata_path, 'w') as json_file:
      json.dump(self.__metadata, json_file, indent=2)

  def __restore_metadata(self):
    """ Read the stored json file and return the time stamps of the

        metadata file.
        """
    if not os.path.exists(self.__combined_metadata_path):
      return None

    with open(self.__combined_metadata_path, 'r') as json_file:
      try:
        self.__metadata = json.load(json_file)
      except json.decoder.JSONDecodeError:
        logging.warning('JSONDecodeError!!!: skip reading the {} file'.format(
            self.__combined_metadata_path))
        return None

    logging.info('Metadata restored from {}'.format(
        self.__combined_metadata_path))
    self.__update_export_map()
    return os.path.getmtime(self.__combined_metadata_path)


def get_args():

  def check_dir(path):
    if os.path.exists(path) and os.path.isdir(path):
      return os.path.normpath(path)
    else:
      raise argparse.ArgumentTypeError('\"{}\" is not a directory'.format(path))

  parser = argparse.ArgumentParser()
  parser.add_argument(
      '--component-top',
      help='Scan all components under this directory.',
      default=os.path.join(os.path.dirname(__file__), '../../../components'),
      type=check_dir)
  parser.add_argument(
      '--meta-file',
      help='Name of the metadata file.',
      default=COMPONENT_METADATA_FILE,
      type=str)
  parser.add_argument(
      '--meta-dir',
      help='Each component has the metadata in this directory.',
      default=COMPONENT_METADATA_DIR,
      type=str)
  parser.add_argument(
      '--out-dir',
      help='Out dir for the outer tree. The orchestrator stores the collected metadata in this directory.',
      default=os.path.join(os.path.dirname(__file__), '../../../out'),
      type=os.path.normpath)
  parser.add_argument(
      '--force',
      '-f',
      action='store_true',
      help='Force to collect metadata',
  )
  parser.add_argument(
      '--verbose',
      '-v',
      help='Increase output verbosity, e.g. "-v", "-vv".',
      action='count',
      default=0)
  return parser.parse_args()


def main():
  args = get_args()
  utils.set_logging_config(args.verbose)

  metadata_collector = MetadataCollector(args.component_top, args.out_dir,
                                         args.meta_dir, args.meta_file, args.force)
  metadata_collector.collect()


if __name__ == '__main__':
  main()