scripts/hiddenapi/verify_overlaps.py

#!/usr/bin/env python
#
# Copyright (C) 2018 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Verify that one set of hidden API flags is a subset of another.
"""

import argparse
import csv
import sys
from itertools import chain

class InteriorNode:
    """
    An interior node in a trie.

    Each interior node has a dict that maps from an element of a signature to
    either another interior node or a leaf. Each interior node represents either
    a package, class or nested class. Class members are represented by a Leaf.

    Associating the set of flags [public-api] with the signature
    "Ljava/lang/Object;->String()Ljava/lang/String;" will cause the following
    nodes to be created:
    Node()
    ^- package:java -> Node()
       ^- package:lang -> Node()
           ^- class:Object -> Node()
              ^- member:String()Ljava/lang/String; -> Leaf([public-api])

    Associating the set of flags [blocked,core-platform-api] with the signature
    "Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;"
    will cause the following nodes to be created:
    Node()
    ^- package:java -> Node()
       ^- package:lang -> Node()
           ^- class:Character -> Node()
              ^- class:UnicodeScript -> Node()
                 ^- member:of(I)Ljava/lang/Character$UnicodeScript;
                    -> Leaf([blocked,core-platform-api])

    Attributes:
        nodes: a dict from an element of the signature to the Node/Leaf
        containing the next element/value.
    """
    def __init__(self):
        self.nodes = {}

    def signatureToElements(self, signature):
        """
        Split a signature or a prefix into a number of elements:
        1. The packages (excluding the leading L preceding the first package).
        2. The class names, from outermost to innermost.
        3. The member signature.

        e.g. Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;
        will be broken down into these elements:
        1. package:java
        2. package:lang
        3. class:Character
        4. class:UnicodeScript
        5. member:of(I)Ljava/lang/Character$UnicodeScript;
        """
        # Remove the leading L.
        #  - java/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;
        text = signature.removeprefix("L")
        # Split the signature between qualified class name and the class member
        # signature.
        #  0 - java/lang/Character$UnicodeScript
        #  1 - of(I)Ljava/lang/Character$UnicodeScript;
        parts = text.split(";->")
        member = parts[1:]
        # Split the qualified class name into packages, and class name.
        #  0 - java
        #  1 - lang
        #  2 - Character$UnicodeScript
        elements = parts[0].split("/")
        packages = elements[0:-1]
        className = elements[-1]
        if className == "*" or className == "**":
            # Cannot specify a wildcard and target a specific member
            if len(member) != 0:
                raise Exception("Invalid signature %s: contains wildcard %s and member signature %s"
                                % (signature, className, member[0]))
            wildcard = [className]
            # Assemble the parts into a single list, adding prefixes to identify
            # the different parts.
            #  0 - package:java
            #  1 - package:lang
            #  2 - *
            return list(chain(map(lambda x : "package:" + x, packages),
                              wildcard))
        else:
            # Split the class name into outer / inner classes
            #  0 - Character
            #  1 - UnicodeScript
            classes = className.split("$")
            # Assemble the parts into a single list, adding prefixes to identify
            # the different parts.
            #  0 - package:java
            #  1 - package:lang
            #  2 - class:Character
            #  3 - class:UnicodeScript
            #  4 - member:of(I)Ljava/lang/Character$UnicodeScript;
            return list(chain(map(lambda x : "package:" + x, packages),
                              map(lambda x : "class:" + x, classes),
                              map(lambda x : "member:" + x, member)))

    def add(self, signature, value):
        """
        Associate the value with the specific signature.
        :param signature: the member signature
        :param value: the value to associated with the signature
        :return: n/a
        """
        # Split the signature into elements.
        elements = self.signatureToElements(signature)
        # Find the Node associated with the deepest class.
        node = self
        for element in elements[:-1]:
            if element in node.nodes:
                node = node.nodes[element]
            else:
                next = InteriorNode()
                node.nodes[element] = next
                node = next
        # Add a Leaf containing the value and associate it with the member
        # signature within the class.
        lastElement = elements[-1]
        if not lastElement.startswith("member:"):
            raise Exception("Invalid signature: %s, does not identify a specific member" % signature)
        if lastElement in node.nodes:
            raise Exception("Duplicate signature: %s" % signature)
        node.nodes[lastElement] = Leaf(value)

    def getMatchingRows(self, pattern):
        """
        Get the values (plural) associated with the pattern.

        e.g. If the pattern is a full signature then this will return a list
        containing the value associated with that signature.

        If the pattern is a class then this will return a list containing the
        values associated with all members of that class.

        If the pattern is a package then this will return a list containing the
        values associated with all the members of all the classes in that
        package and sub-packages.

        If the pattern ends with "*" then the preceding part is treated as a
        package and this will return a list containing the values associated
        with all the members of all the classes in that package.

        If the pattern ends with "**" then the preceding part is treated
        as a package and this will return a list containing the values
        associated with all the members of all the classes in that package and
        all sub-packages.

        :param pattern: the pattern which could be a complete signature or a
        class, or package wildcard.
        :return: an iterable containing all the values associated with the
        pattern.
        """
        elements = self.signatureToElements(pattern)
        node = self
        # Include all values from this node and all its children.
        selector = lambda x : True
        lastElement = elements[-1]
        if lastElement == "*" or lastElement == "**":
            elements = elements[:-1]
            if lastElement == "*":
                # Do not include values from sub-packages.
                selector = lambda x : not x.startswith("package:")
        for element in elements:
            if element in node.nodes:
                node = node.nodes[element]
            else:
                return []
        return chain.from_iterable(node.values(selector))

    def values(self, selector):
        """
        :param selector: a function that can be applied to a key in the nodes
        attribute to determine whether to return its values.
        :return: A list of iterables of all the values associated with this
        node and its children.
        """
        values = []
        self.appendValues(values, selector)
        return values

    def appendValues(self, values, selector):
        """
        Append the values associated with this node and its children to the
        list.

        For each item (key, child) in nodes the child node's values are returned
        if and only if the selector returns True when called on its key. A child
        node's values are all the values associated with it and all its
        descendant nodes.

        :param selector: a function that can be applied to a key in the nodes
        attribute to determine whether to return its values.
        :param values: a list of a iterables of values.
        """
        for key, node in self.nodes.items():
            if selector(key):
                node.appendValues(values, lambda x : True)

class Leaf:
    """
    A leaf of the trie

    Attributes:
        value: the value associated with this leaf.
    """
    def __init__(self, value):
        self.value = value

    def values(self, selector):
        """
        :return: A list of a list of the value associated with this node.
        """
        return [[self.value]]

    def appendValues(self, values, selector):
        """
        Appends a list of the value associated with this node to the list.
        :param values: a list of a iterables of values.
        """
        values.append([self.value])

def dict_reader(input):
    return csv.DictReader(input, delimiter=',', quotechar='|', fieldnames=['signature'])

def read_flag_trie_from_file(file):
    with open(file, 'r') as stream:
        return read_flag_trie_from_stream(stream)

def read_flag_trie_from_stream(stream):
    trie = InteriorNode()
    reader = dict_reader(stream)
    for row in reader:
        signature = row['signature']
        trie.add(signature, row)
    return trie

def extract_subset_from_monolithic_flags_as_dict_from_file(monolithicTrie, patternsFile):
    """
    Extract a subset of flags from the dict containing all the monolithic flags.

    :param monolithicFlagsDict: the dict containing all the monolithic flags.
    :param patternsFile: a file containing a list of signature patterns that
    define the subset.
    :return: the dict from signature to row.
    """
    with open(patternsFile, 'r') as stream:
        return extract_subset_from_monolithic_flags_as_dict_from_stream(monolithicTrie, stream)

def extract_subset_from_monolithic_flags_as_dict_from_stream(monolithicTrie, stream):
    """
    Extract a subset of flags from the trie containing all the monolithic flags.

    :param monolithicTrie: the trie containing all the monolithic flags.
    :param stream: a stream containing a list of signature patterns that define
    the subset.
    :return: the dict from signature to row.
    """
    dict = {}
    for pattern in stream:
        pattern = pattern.rstrip()
        rows = monolithicTrie.getMatchingRows(pattern)
        for row in rows:
            signature = row['signature']
            dict[signature] = row
    return dict

def read_signature_csv_from_stream_as_dict(stream):
    """
    Read the csv contents from the stream into a dict. The first column is assumed to be the
    signature and used as the key. The whole row is stored as the value.

    :param stream: the csv contents to read
    :return: the dict from signature to row.
    """
    dict = {}
    reader = dict_reader(stream)
    for row in reader:
        signature = row['signature']
        dict[signature] = row
    return dict

def read_signature_csv_from_file_as_dict(csvFile):
    """
    Read the csvFile into a dict. The first column is assumed to be the
    signature and used as the key. The whole row is stored as the value.

    :param csvFile: the csv file to read
    :return: the dict from signature to row.
    """
    with open(csvFile, 'r') as f:
        return read_signature_csv_from_stream_as_dict(f)

def compare_signature_flags(monolithicFlagsDict, modularFlagsDict):
    """
    Compare the signature flags between the two dicts.

    :param monolithicFlagsDict: the dict containing the subset of the monolithic
    flags that should be equal to the modular flags.
    :param modularFlagsDict:the dict containing the flags produced by a single
    bootclasspath_fragment module.
    :return: list of mismatches., each mismatch is a tuple where the first item
    is the signature, and the second and third items are lists of the flags from
    modular dict, and monolithic dict respectively.
    """
    mismatchingSignatures = []
    # Create a sorted set of all the signatures from both the monolithic and
    # modular dicts.
    allSignatures = sorted(set(chain(monolithicFlagsDict.keys(), modularFlagsDict.keys())))
    for signature in allSignatures:
        monolithicRow = monolithicFlagsDict.get(signature, {})
        monolithicFlags = monolithicRow.get(None, [])
        modularRow = modularFlagsDict.get(signature, {})
        modularFlags = modularRow.get(None, [])
        if monolithicFlags != modularFlags:
            mismatchingSignatures.append((signature, modularFlags, monolithicFlags))
    return mismatchingSignatures

def main(argv):
    args_parser = argparse.ArgumentParser(description='Verify that sets of hidden API flags are each a subset of the monolithic flag file.')
    args_parser.add_argument('monolithicFlags', help='The monolithic flag file')
    args_parser.add_argument('modularFlags', nargs=argparse.REMAINDER, help='Flags produced by individual bootclasspath_fragment modules')
    args = args_parser.parse_args(argv[1:])

    # Read in all the flags into the trie
    monolithicFlagsPath = args.monolithicFlags
    monolithicTrie = read_flag_trie_from_file(monolithicFlagsPath)

    # For each subset specified on the command line, create dicts for the flags
    # provided by the subset and the corresponding flags from the complete set
    # of flags and compare them.
    failed = False
    for modularPair in args.modularFlags:
        parts = modularPair.split(":")
        modularFlagsPath = parts[0]
        modularPatternsPath = parts[1]
        modularFlagsDict = read_signature_csv_from_file_as_dict(modularFlagsPath)
        monolithicFlagsSubsetDict = extract_subset_from_monolithic_flags_as_dict_from_file(monolithicTrie, modularPatternsPath)
        mismatchingSignatures = compare_signature_flags(monolithicFlagsSubsetDict, modularFlagsDict)
        if mismatchingSignatures:
            failed = True
            print("ERROR: Hidden API flags are inconsistent:")
            print("< " + modularFlagsPath)
            print("> " + monolithicFlagsPath)
            for mismatch in mismatchingSignatures:
                signature = mismatch[0]
                print()
                print("< " + ",".join([signature]+ mismatch[1]))
                print("> " + ",".join([signature]+ mismatch[2]))

    if failed:
        sys.exit(1)

if __name__ == "__main__":
    main(sys.argv)