| #!/usr/bin/python |
| """A glorified C pre-processor parser.""" |
| |
| import ctypes |
| import logging |
| import os |
| import re |
| import site |
| import utils |
| |
| top = os.getenv('ANDROID_BUILD_TOP') |
| if top is None: |
| utils.panic('ANDROID_BUILD_TOP not set.\n') |
| |
| # Set up the env vars for libclang. |
| site.addsitedir(os.path.join(top, 'external/clang/bindings/python')) |
| |
| import clang.cindex |
| from clang.cindex import conf |
| from clang.cindex import Cursor |
| from clang.cindex import CursorKind |
| from clang.cindex import SourceLocation |
| from clang.cindex import SourceRange |
| from clang.cindex import TokenGroup |
| from clang.cindex import TokenKind |
| from clang.cindex import TranslationUnit |
| |
| # Set up LD_LIBRARY_PATH to include libclang.so, libLLVM.so, and etc. |
| # Note that setting LD_LIBRARY_PATH with os.putenv() sometimes doesn't help. |
| clang.cindex.Config.set_library_path(os.path.join(top, 'prebuilts/sdk/tools/linux/lib64')) |
| |
| from defaults import kCppUndefinedMacro |
| from defaults import kernel_remove_config_macros |
| from defaults import kernel_token_replacements |
| |
| |
| debugBlockParser = False |
| debugCppExpr = False |
| debugOptimIf01 = False |
| |
| ############################################################################### |
| ############################################################################### |
| ##### ##### |
| ##### C P P T O K E N S ##### |
| ##### ##### |
| ############################################################################### |
| ############################################################################### |
| |
| # the list of supported C-preprocessor tokens |
| # plus a couple of C tokens as well |
| tokEOF = "\0" |
| tokLN = "\n" |
| tokSTRINGIFY = "#" |
| tokCONCAT = "##" |
| tokLOGICAND = "&&" |
| tokLOGICOR = "||" |
| tokSHL = "<<" |
| tokSHR = ">>" |
| tokEQUAL = "==" |
| tokNEQUAL = "!=" |
| tokLT = "<" |
| tokLTE = "<=" |
| tokGT = ">" |
| tokGTE = ">=" |
| tokELLIPSIS = "..." |
| tokSPACE = " " |
| tokDEFINED = "defined" |
| tokLPAREN = "(" |
| tokRPAREN = ")" |
| tokNOT = "!" |
| tokPLUS = "+" |
| tokMINUS = "-" |
| tokMULTIPLY = "*" |
| tokDIVIDE = "/" |
| tokMODULUS = "%" |
| tokBINAND = "&" |
| tokBINOR = "|" |
| tokBINXOR = "^" |
| tokCOMMA = "," |
| tokLBRACE = "{" |
| tokRBRACE = "}" |
| tokARROW = "->" |
| tokINCREMENT = "++" |
| tokDECREMENT = "--" |
| tokNUMBER = "<number>" |
| tokIDENT = "<ident>" |
| tokSTRING = "<string>" |
| |
| |
| class Token(clang.cindex.Token): |
| """A class that represents one token after parsing. |
| |
| It inherits the class in libclang, with an extra id property to hold the |
| new spelling of the token. The spelling property in the base class is |
| defined as read-only. New names after macro instantiation are saved in |
| their ids now. It also facilitates the renaming of directive optimizations |
| like replacing 'ifndef X' with 'if !defined(X)'. |
| |
| It also overrides the cursor property of the base class. Because the one |
| in libclang always queries based on a single token, which usually doesn't |
| hold useful information. The cursor in this class can be set by calling |
| CppTokenizer.getTokensWithCursors(). Otherwise it returns the one in the |
| base class. |
| """ |
| |
| def __init__(self, tu=None, group=None, int_data=None, ptr_data=None, |
| cursor=None): |
| clang.cindex.Token.__init__(self) |
| self._id = None |
| self._tu = tu |
| self._group = group |
| self._cursor = cursor |
| # self.int_data and self.ptr_data are from the base class. But |
| # self.int_data doesn't accept a None value. |
| if int_data is not None: |
| self.int_data = int_data |
| self.ptr_data = ptr_data |
| |
| @property |
| def id(self): |
| """Name of the token.""" |
| if self._id is None: |
| return self.spelling |
| else: |
| return self._id |
| |
| @id.setter |
| def id(self, new_id): |
| """Setting name of the token.""" |
| self._id = new_id |
| |
| @property |
| def cursor(self): |
| if self._cursor is None: |
| self._cursor = clang.cindex.Token.cursor |
| return self._cursor |
| |
| @cursor.setter |
| def cursor(self, new_cursor): |
| self._cursor = new_cursor |
| |
| def __repr__(self): |
| if self.id == 'defined': |
| return self.id |
| elif self.kind == TokenKind.IDENTIFIER: |
| return "(ident %s)" % self.id |
| |
| return self.id |
| |
| def __str__(self): |
| return self.id |
| |
| |
| class BadExpectedToken(Exception): |
| """An exception that will be raised for unexpected tokens.""" |
| pass |
| |
| |
| # The __contains__ function in libclang SourceRange class contains a bug. It |
| # gives wrong result when dealing with single line range. |
| # Bug filed with upstream: |
| # http://llvm.org/bugs/show_bug.cgi?id=22243, http://reviews.llvm.org/D7277 |
| def SourceRange__contains__(self, other): |
| """Determine if a given location is inside the range.""" |
| if not isinstance(other, SourceLocation): |
| return False |
| if other.file is None and self.start.file is None: |
| pass |
| elif (self.start.file.name != other.file.name or |
| other.file.name != self.end.file.name): |
| # same file name |
| return False |
| # same file, in between lines |
| if self.start.line < other.line < self.end.line: |
| return True |
| # same file, same line |
| elif self.start.line == other.line == self.end.line: |
| if self.start.column <= other.column <= self.end.column: |
| return True |
| elif self.start.line == other.line: |
| # same file first line |
| if self.start.column <= other.column: |
| return True |
| elif other.line == self.end.line: |
| # same file last line |
| if other.column <= self.end.column: |
| return True |
| return False |
| |
| |
| SourceRange.__contains__ = SourceRange__contains__ |
| |
| |
| ################################################################################ |
| ################################################################################ |
| ##### ##### |
| ##### C P P T O K E N I Z E R ##### |
| ##### ##### |
| ################################################################################ |
| ################################################################################ |
| |
| |
| class CppTokenizer(object): |
| """A tokenizer that converts some input text into a list of tokens. |
| |
| It calls libclang's tokenizer to get the parsed tokens. In addition, it |
| updates the cursor property in each token after parsing, by calling |
| getTokensWithCursors(). |
| """ |
| |
| clang_flags = ['-E', '-x', 'c'] |
| options = TranslationUnit.PARSE_DETAILED_PROCESSING_RECORD |
| |
| def __init__(self): |
| """Initialize a new CppTokenizer object.""" |
| self._indexer = clang.cindex.Index.create() |
| self._tu = None |
| self._index = 0 |
| self.tokens = None |
| |
| def _getTokensWithCursors(self): |
| """Helper method to return all tokens with their cursors. |
| |
| The cursor property in a clang Token doesn't provide enough |
| information. Because it is queried based on single token each time |
| without any context, i.e. via calling conf.lib.clang_annotateTokens() |
| with only one token given. So we often see 'INVALID_FILE' in one |
| token's cursor. In this function it passes all the available tokens |
| to get more informative cursors. |
| """ |
| |
| tokens_memory = ctypes.POINTER(clang.cindex.Token)() |
| tokens_count = ctypes.c_uint() |
| |
| conf.lib.clang_tokenize(self._tu, self._tu.cursor.extent, |
| ctypes.byref(tokens_memory), |
| ctypes.byref(tokens_count)) |
| |
| count = int(tokens_count.value) |
| |
| # If we get no tokens, no memory was allocated. Be sure not to return |
| # anything and potentially call a destructor on nothing. |
| if count < 1: |
| return |
| |
| cursors = (Cursor * count)() |
| cursors_memory = ctypes.cast(cursors, ctypes.POINTER(Cursor)) |
| |
| conf.lib.clang_annotateTokens(self._tu, tokens_memory, count, |
| cursors_memory) |
| |
| tokens_array = ctypes.cast( |
| tokens_memory, |
| ctypes.POINTER(clang.cindex.Token * count)).contents |
| token_group = TokenGroup(self._tu, tokens_memory, tokens_count) |
| |
| tokens = [] |
| for i in xrange(0, count): |
| token = Token(self._tu, token_group, |
| int_data=tokens_array[i].int_data, |
| ptr_data=tokens_array[i].ptr_data, |
| cursor=cursors[i]) |
| # We only want non-comment tokens. |
| if token.kind != TokenKind.COMMENT: |
| tokens.append(token) |
| |
| return tokens |
| |
| def parseString(self, lines): |
| """Parse a list of text lines into a BlockList object.""" |
| file_ = 'dummy.c' |
| self._tu = self._indexer.parse(file_, self.clang_flags, |
| unsaved_files=[(file_, lines)], |
| options=self.options) |
| self.tokens = self._getTokensWithCursors() |
| |
| def parseFile(self, file_): |
| """Parse a file into a BlockList object.""" |
| self._tu = self._indexer.parse(file_, self.clang_flags, |
| options=self.options) |
| self.tokens = self._getTokensWithCursors() |
| |
| def nextToken(self): |
| """Return next token from the list.""" |
| if self._index < len(self.tokens): |
| t = self.tokens[self._index] |
| self._index += 1 |
| return t |
| else: |
| return None |
| |
| |
| class CppStringTokenizer(CppTokenizer): |
| """A CppTokenizer derived class that accepts a string of text as input.""" |
| |
| def __init__(self, line): |
| CppTokenizer.__init__(self) |
| self.parseString(line) |
| |
| |
| class CppFileTokenizer(CppTokenizer): |
| """A CppTokenizer derived class that accepts a file as input.""" |
| |
| def __init__(self, file_): |
| CppTokenizer.__init__(self) |
| self.parseFile(file_) |
| |
| |
| # Unit testing |
| # |
| class CppTokenizerTester(object): |
| """A class used to test CppTokenizer classes.""" |
| |
| def __init__(self, tokenizer=None): |
| self._tokenizer = tokenizer |
| self._token = None |
| |
| def setTokenizer(self, tokenizer): |
| self._tokenizer = tokenizer |
| |
| def expect(self, id): |
| self._token = self._tokenizer.nextToken() |
| if self._token is None: |
| tokid = '' |
| else: |
| tokid = self._token.id |
| if tokid == id: |
| return |
| raise BadExpectedToken("### BAD TOKEN: '%s' expecting '%s'" % ( |
| tokid, id)) |
| |
| def expectToken(self, id, line, col): |
| self.expect(id) |
| if self._token.location.line != line: |
| raise BadExpectedToken( |
| "### BAD LINENO: token '%s' got '%d' expecting '%d'" % ( |
| id, self._token.lineno, line)) |
| if self._token.location.column != col: |
| raise BadExpectedToken("### BAD COLNO: '%d' expecting '%d'" % ( |
| self._token.colno, col)) |
| |
| def expectTokens(self, tokens): |
| for id, line, col in tokens: |
| self.expectToken(id, line, col) |
| |
| def expectList(self, list_): |
| for item in list_: |
| self.expect(item) |
| |
| |
| def test_CppTokenizer(): |
| tester = CppTokenizerTester() |
| |
| tester.setTokenizer(CppStringTokenizer("#an/example && (01923_xy)")) |
| tester.expectList(["#", "an", "/", "example", tokLOGICAND, tokLPAREN, |
| "01923_xy", tokRPAREN]) |
| |
| tester.setTokenizer(CppStringTokenizer("FOO(BAR) && defined(BAZ)")) |
| tester.expectList(["FOO", tokLPAREN, "BAR", tokRPAREN, tokLOGICAND, |
| "defined", tokLPAREN, "BAZ", tokRPAREN]) |
| |
| tester.setTokenizer(CppStringTokenizer("/*\n#\n*/")) |
| tester.expectList([]) |
| |
| tester.setTokenizer(CppStringTokenizer("first\nsecond")) |
| tester.expectList(["first", "second"]) |
| |
| tester.setTokenizer(CppStringTokenizer("first second\n third")) |
| tester.expectTokens([("first", 1, 1), |
| ("second", 1, 7), |
| ("third", 2, 3)]) |
| |
| tester.setTokenizer(CppStringTokenizer("boo /* what the\nhell */")) |
| tester.expectTokens([("boo", 1, 1)]) |
| |
| tester.setTokenizer(CppStringTokenizer("an \\\n example")) |
| tester.expectTokens([("an", 1, 1), |
| ("example", 2, 2)]) |
| return True |
| |
| |
| ################################################################################ |
| ################################################################################ |
| ##### ##### |
| ##### C P P E X P R E S S I O N S ##### |
| ##### ##### |
| ################################################################################ |
| ################################################################################ |
| |
| |
| class CppExpr(object): |
| """A class that models the condition of #if directives into an expr tree. |
| |
| Each node in the tree is of the form (op, arg) or (op, arg1, arg2) where |
| "op" is a string describing the operation |
| """ |
| |
| unaries = ["!", "~"] |
| binaries = ["+", "-", "<", "<=", ">=", ">", "&&", "||", "*", "/", "%", |
| "&", "|", "^", "<<", ">>", "==", "!=", "?", ":"] |
| precedences = { |
| "?": 1, ":": 1, |
| "||": 2, |
| "&&": 3, |
| "|": 4, |
| "^": 5, |
| "&": 6, |
| "==": 7, "!=": 7, |
| "<": 8, "<=": 8, ">": 8, ">=": 8, |
| "<<": 9, ">>": 9, |
| "+": 10, "-": 10, |
| "*": 11, "/": 11, "%": 11, |
| "!": 12, "~": 12 |
| } |
| |
| def __init__(self, tokens): |
| """Initialize a CppExpr. 'tokens' must be a CppToken list.""" |
| self.tokens = tokens |
| self._num_tokens = len(tokens) |
| self._index = 0 |
| |
| if debugCppExpr: |
| print "CppExpr: trying to parse %s" % repr(tokens) |
| self.expr = self.parseExpression(0) |
| if debugCppExpr: |
| print "CppExpr: got " + repr(self.expr) |
| if self._index != self._num_tokens: |
| self.throw(BadExpectedToken, "crap at end of input (%d != %d): %s" |
| % (self._index, self._num_tokens, repr(tokens))) |
| |
| def throw(self, exception, msg): |
| if self._index < self._num_tokens: |
| tok = self.tokens[self._index] |
| print "%d:%d: %s" % (tok.location.line, tok.location.column, msg) |
| else: |
| print "EOF: %s" % msg |
| raise exception(msg) |
| |
| def expectId(self, id): |
| """Check that a given token id is at the current position.""" |
| token = self.tokens[self._index] |
| if self._index >= self._num_tokens or token.id != id: |
| self.throw(BadExpectedToken, |
| "### expecting '%s' in expression, got '%s'" % ( |
| id, token.id)) |
| self._index += 1 |
| |
| def is_decimal(self): |
| token = self.tokens[self._index].id |
| if token[-1] in "ULul": |
| token = token[:-1] |
| try: |
| val = int(token, 10) |
| self._index += 1 |
| return ('int', val) |
| except ValueError: |
| return None |
| |
| def is_octal(self): |
| token = self.tokens[self._index].id |
| if token[-1] in "ULul": |
| token = token[:-1] |
| if len(token) < 2 or token[0] != '0': |
| return None |
| try: |
| val = int(token, 8) |
| self._index += 1 |
| return ('oct', val) |
| except ValueError: |
| return None |
| |
| def is_hexadecimal(self): |
| token = self.tokens[self._index].id |
| if token[-1] in "ULul": |
| token = token[:-1] |
| if len(token) < 3 or (token[:2] != '0x' and token[:2] != '0X'): |
| return None |
| try: |
| val = int(token, 16) |
| self._index += 1 |
| return ('hex', val) |
| except ValueError: |
| return None |
| |
| def is_integer(self): |
| if self.tokens[self._index].kind != TokenKind.LITERAL: |
| return None |
| |
| c = self.is_hexadecimal() |
| if c: |
| return c |
| |
| c = self.is_octal() |
| if c: |
| return c |
| |
| c = self.is_decimal() |
| if c: |
| return c |
| |
| return None |
| |
| def is_number(self): |
| t = self.tokens[self._index] |
| if t.id == tokMINUS and self._index + 1 < self._num_tokens: |
| self._index += 1 |
| c = self.is_integer() |
| if c: |
| op, val = c |
| return (op, -val) |
| if t.id == tokPLUS and self._index + 1 < self._num_tokens: |
| self._index += 1 |
| c = self.is_integer() |
| if c: |
| return c |
| |
| return self.is_integer() |
| |
| def is_defined(self): |
| t = self.tokens[self._index] |
| if t.id != tokDEFINED: |
| return None |
| |
| # We have the defined keyword, check the rest. |
| self._index += 1 |
| used_parens = False |
| if (self._index < self._num_tokens and |
| self.tokens[self._index].id == tokLPAREN): |
| used_parens = True |
| self._index += 1 |
| |
| if self._index >= self._num_tokens: |
| self.throw(BadExpectedToken, |
| "### 'defined' must be followed by macro name or left " |
| "paren") |
| |
| t = self.tokens[self._index] |
| if t.kind != TokenKind.IDENTIFIER: |
| self.throw(BadExpectedToken, |
| "### 'defined' must be followed by macro name") |
| |
| self._index += 1 |
| if used_parens: |
| self.expectId(tokRPAREN) |
| |
| return ("defined", t.id) |
| |
| def is_call_or_ident(self): |
| if self._index >= self._num_tokens: |
| return None |
| |
| t = self.tokens[self._index] |
| if t.kind != TokenKind.IDENTIFIER: |
| return None |
| |
| name = t.id |
| |
| self._index += 1 |
| if (self._index >= self._num_tokens or |
| self.tokens[self._index].id != tokLPAREN): |
| return ("ident", name) |
| |
| params = [] |
| depth = 1 |
| self._index += 1 |
| j = self._index |
| while self._index < self._num_tokens: |
| id = self.tokens[self._index].id |
| if id == tokLPAREN: |
| depth += 1 |
| elif depth == 1 and (id == tokCOMMA or id == tokRPAREN): |
| k = self._index |
| param = self.tokens[j:k] |
| params.append(param) |
| if id == tokRPAREN: |
| break |
| j = self._index + 1 |
| elif id == tokRPAREN: |
| depth -= 1 |
| self._index += 1 |
| |
| if self._index >= self._num_tokens: |
| return None |
| |
| self._index += 1 |
| return ("call", (name, params)) |
| |
| # Implements the "precedence climbing" algorithm from |
| # http://www.engr.mun.ca/~theo/Misc/exp_parsing.htm. |
| # The "classic" algorithm would be fine if we were using a tool to |
| # generate the parser, but we're not. Dijkstra's "shunting yard" |
| # algorithm hasn't been necessary yet. |
| |
| def parseExpression(self, minPrecedence): |
| if self._index >= self._num_tokens: |
| return None |
| |
| node = self.parsePrimary() |
| while (self.token() and self.isBinary(self.token()) and |
| self.precedence(self.token()) >= minPrecedence): |
| op = self.token() |
| self.nextToken() |
| rhs = self.parseExpression(self.precedence(op) + 1) |
| node = (op.id, node, rhs) |
| |
| return node |
| |
| def parsePrimary(self): |
| op = self.token() |
| if self.isUnary(op): |
| self.nextToken() |
| return (op.id, self.parseExpression(self.precedence(op))) |
| |
| primary = None |
| if op.id == tokLPAREN: |
| self.nextToken() |
| primary = self.parseExpression(0) |
| self.expectId(tokRPAREN) |
| elif op.id == "?": |
| self.nextToken() |
| primary = self.parseExpression(0) |
| self.expectId(":") |
| elif op.id == '+' or op.id == '-' or op.kind == TokenKind.LITERAL: |
| primary = self.is_number() |
| # Checking for 'defined' needs to come first now because 'defined' is |
| # recognized as IDENTIFIER. |
| elif op.id == tokDEFINED: |
| primary = self.is_defined() |
| elif op.kind == TokenKind.IDENTIFIER: |
| primary = self.is_call_or_ident() |
| else: |
| self.throw(BadExpectedToken, |
| "didn't expect to see a %s in factor" % ( |
| self.tokens[self._index].id)) |
| return primary |
| |
| def isBinary(self, token): |
| return token.id in self.binaries |
| |
| def isUnary(self, token): |
| return token.id in self.unaries |
| |
| def precedence(self, token): |
| return self.precedences.get(token.id) |
| |
| def token(self): |
| if self._index >= self._num_tokens: |
| return None |
| return self.tokens[self._index] |
| |
| def nextToken(self): |
| self._index += 1 |
| if self._index >= self._num_tokens: |
| return None |
| return self.tokens[self._index] |
| |
| def dump_node(self, e): |
| op = e[0] |
| line = "(" + op |
| if op == "int": |
| line += " %d)" % e[1] |
| elif op == "oct": |
| line += " 0%o)" % e[1] |
| elif op == "hex": |
| line += " 0x%x)" % e[1] |
| elif op == "ident": |
| line += " %s)" % e[1] |
| elif op == "defined": |
| line += " %s)" % e[1] |
| elif op == "call": |
| arg = e[1] |
| line += " %s [" % arg[0] |
| prefix = "" |
| for param in arg[1]: |
| par = "" |
| for tok in param: |
| par += str(tok) |
| line += "%s%s" % (prefix, par) |
| prefix = "," |
| line += "])" |
| elif op in CppExpr.unaries: |
| line += " %s)" % self.dump_node(e[1]) |
| elif op in CppExpr.binaries: |
| line += " %s %s)" % (self.dump_node(e[1]), self.dump_node(e[2])) |
| else: |
| line += " ?%s)" % repr(e[1]) |
| |
| return line |
| |
| def __repr__(self): |
| return self.dump_node(self.expr) |
| |
| def source_node(self, e): |
| op = e[0] |
| if op == "int": |
| return "%d" % e[1] |
| if op == "hex": |
| return "0x%x" % e[1] |
| if op == "oct": |
| return "0%o" % e[1] |
| if op == "ident": |
| # XXX: should try to expand |
| return e[1] |
| if op == "defined": |
| return "defined(%s)" % e[1] |
| |
| prec = CppExpr.precedences.get(op, 1000) |
| arg = e[1] |
| if op in CppExpr.unaries: |
| arg_src = self.source_node(arg) |
| arg_op = arg[0] |
| arg_prec = CppExpr.precedences.get(arg_op, 1000) |
| if arg_prec < prec: |
| return "!(" + arg_src + ")" |
| else: |
| return "!" + arg_src |
| if op in CppExpr.binaries: |
| arg2 = e[2] |
| arg1_op = arg[0] |
| arg2_op = arg2[0] |
| arg1_src = self.source_node(arg) |
| arg2_src = self.source_node(arg2) |
| if CppExpr.precedences.get(arg1_op, 1000) < prec: |
| arg1_src = "(%s)" % arg1_src |
| if CppExpr.precedences.get(arg2_op, 1000) < prec: |
| arg2_src = "(%s)" % arg2_src |
| |
| return "%s %s %s" % (arg1_src, op, arg2_src) |
| return "???" |
| |
| def __str__(self): |
| return self.source_node(self.expr) |
| |
| @staticmethod |
| def int_node(e): |
| if e[0] in ["int", "oct", "hex"]: |
| return e[1] |
| else: |
| return None |
| |
| def toInt(self): |
| return self.int_node(self.expr) |
| |
| def optimize_node(self, e, macros=None): |
| if macros is None: |
| macros = {} |
| op = e[0] |
| |
| if op == "defined": |
| op, name = e |
| if macros.has_key(name): |
| if macros[name] == kCppUndefinedMacro: |
| return ("int", 0) |
| else: |
| try: |
| value = int(macros[name]) |
| return ("int", value) |
| except ValueError: |
| return ("defined", macros[name]) |
| |
| if kernel_remove_config_macros and name.startswith("CONFIG_"): |
| return ("int", 0) |
| |
| return e |
| |
| elif op == "ident": |
| op, name = e |
| if macros.has_key(name): |
| try: |
| value = int(macros[name]) |
| expanded = ("int", value) |
| except ValueError: |
| expanded = ("ident", macros[name]) |
| return self.optimize_node(expanded, macros) |
| return e |
| |
| elif op == "!": |
| op, v = e |
| v = self.optimize_node(v, macros) |
| if v[0] == "int": |
| if v[1] == 0: |
| return ("int", 1) |
| else: |
| return ("int", 0) |
| return ('!', v) |
| |
| elif op == "&&": |
| op, l, r = e |
| l = self.optimize_node(l, macros) |
| r = self.optimize_node(r, macros) |
| li = self.int_node(l) |
| ri = self.int_node(r) |
| if li is not None: |
| if li == 0: |
| return ("int", 0) |
| else: |
| return r |
| elif ri is not None: |
| if ri == 0: |
| return ("int", 0) |
| else: |
| return l |
| return (op, l, r) |
| |
| elif op == "||": |
| op, l, r = e |
| l = self.optimize_node(l, macros) |
| r = self.optimize_node(r, macros) |
| li = self.int_node(l) |
| ri = self.int_node(r) |
| if li is not None: |
| if li == 0: |
| return r |
| else: |
| return ("int", 1) |
| elif ri is not None: |
| if ri == 0: |
| return l |
| else: |
| return ("int", 1) |
| return (op, l, r) |
| |
| else: |
| return e |
| |
| def optimize(self, macros=None): |
| if macros is None: |
| macros = {} |
| self.expr = self.optimize_node(self.expr, macros) |
| |
| |
| def test_cpp_expr(expr, expected): |
| e = CppExpr(CppStringTokenizer(expr).tokens) |
| s1 = repr(e) |
| if s1 != expected: |
| print ("[FAIL]: expression '%s' generates '%s', should be " |
| "'%s'" % (expr, s1, expected)) |
| global failure_count |
| failure_count += 1 |
| |
| |
| def test_cpp_expr_optim(expr, expected, macros=None): |
| if macros is None: |
| macros = {} |
| e = CppExpr(CppStringTokenizer(expr).tokens) |
| e.optimize(macros) |
| s1 = repr(e) |
| if s1 != expected: |
| print ("[FAIL]: optimized expression '%s' generates '%s' with " |
| "macros %s, should be '%s'" % (expr, s1, macros, expected)) |
| global failure_count |
| failure_count += 1 |
| |
| |
| def test_cpp_expr_source(expr, expected): |
| e = CppExpr(CppStringTokenizer(expr).tokens) |
| s1 = str(e) |
| if s1 != expected: |
| print ("[FAIL]: source expression '%s' generates '%s', should " |
| "be '%s'" % (expr, s1, expected)) |
| global failure_count |
| failure_count += 1 |
| |
| |
| def test_CppExpr(): |
| test_cpp_expr("0", "(int 0)") |
| test_cpp_expr("1", "(int 1)") |
| test_cpp_expr("-5", "(int -5)") |
| test_cpp_expr("+1", "(int 1)") |
| test_cpp_expr("0U", "(int 0)") |
| test_cpp_expr("015", "(oct 015)") |
| test_cpp_expr("015l", "(oct 015)") |
| test_cpp_expr("0x3e", "(hex 0x3e)") |
| test_cpp_expr("(0)", "(int 0)") |
| test_cpp_expr("1 && 1", "(&& (int 1) (int 1))") |
| test_cpp_expr("1 && 0", "(&& (int 1) (int 0))") |
| test_cpp_expr("EXAMPLE", "(ident EXAMPLE)") |
| test_cpp_expr("EXAMPLE - 3", "(- (ident EXAMPLE) (int 3))") |
| test_cpp_expr("defined(EXAMPLE)", "(defined EXAMPLE)") |
| test_cpp_expr("defined ( EXAMPLE ) ", "(defined EXAMPLE)") |
| test_cpp_expr("!defined(EXAMPLE)", "(! (defined EXAMPLE))") |
| test_cpp_expr("defined(ABC) || defined(BINGO)", |
| "(|| (defined ABC) (defined BINGO))") |
| test_cpp_expr("FOO(BAR,5)", "(call FOO [BAR,5])") |
| test_cpp_expr("A == 1 || defined(B)", |
| "(|| (== (ident A) (int 1)) (defined B))") |
| |
| test_cpp_expr_optim("0", "(int 0)") |
| test_cpp_expr_optim("1", "(int 1)") |
| test_cpp_expr_optim("1 && 1", "(int 1)") |
| test_cpp_expr_optim("1 && +1", "(int 1)") |
| test_cpp_expr_optim("0x1 && 01", "(oct 01)") |
| test_cpp_expr_optim("1 && 0", "(int 0)") |
| test_cpp_expr_optim("0 && 1", "(int 0)") |
| test_cpp_expr_optim("0 && 0", "(int 0)") |
| test_cpp_expr_optim("1 || 1", "(int 1)") |
| test_cpp_expr_optim("1 || 0", "(int 1)") |
| test_cpp_expr_optim("0 || 1", "(int 1)") |
| test_cpp_expr_optim("0 || 0", "(int 0)") |
| test_cpp_expr_optim("A", "(ident A)") |
| test_cpp_expr_optim("A", "(int 1)", {"A": 1}) |
| test_cpp_expr_optim("A || B", "(int 1)", {"A": 1}) |
| test_cpp_expr_optim("A || B", "(int 1)", {"B": 1}) |
| test_cpp_expr_optim("A && B", "(ident B)", {"A": 1}) |
| test_cpp_expr_optim("A && B", "(ident A)", {"B": 1}) |
| test_cpp_expr_optim("A && B", "(&& (ident A) (ident B))") |
| test_cpp_expr_optim("EXAMPLE", "(ident EXAMPLE)") |
| test_cpp_expr_optim("EXAMPLE - 3", "(- (ident EXAMPLE) (int 3))") |
| test_cpp_expr_optim("defined(EXAMPLE)", "(defined EXAMPLE)") |
| test_cpp_expr_optim("defined(EXAMPLE)", "(defined XOWOE)", |
| {"EXAMPLE": "XOWOE"}) |
| test_cpp_expr_optim("defined(EXAMPLE)", "(int 0)", |
| {"EXAMPLE": kCppUndefinedMacro}) |
| test_cpp_expr_optim("!defined(EXAMPLE)", "(! (defined EXAMPLE))") |
| test_cpp_expr_optim("!defined(EXAMPLE)", "(! (defined XOWOE))", |
| {"EXAMPLE": "XOWOE"}) |
| test_cpp_expr_optim("!defined(EXAMPLE)", "(int 1)", |
| {"EXAMPLE": kCppUndefinedMacro}) |
| test_cpp_expr_optim("defined(A) || defined(B)", |
| "(|| (defined A) (defined B))") |
| test_cpp_expr_optim("defined(A) || defined(B)", "(int 1)", {"A": "1"}) |
| test_cpp_expr_optim("defined(A) || defined(B)", "(int 1)", {"B": "1"}) |
| test_cpp_expr_optim("defined(A) || defined(B)", "(defined A)", |
| {"B": kCppUndefinedMacro}) |
| test_cpp_expr_optim("defined(A) || defined(B)", "(int 0)", |
| {"A": kCppUndefinedMacro, "B": kCppUndefinedMacro}) |
| test_cpp_expr_optim("defined(A) && defined(B)", |
| "(&& (defined A) (defined B))") |
| test_cpp_expr_optim("defined(A) && defined(B)", |
| "(defined B)", {"A": "1"}) |
| test_cpp_expr_optim("defined(A) && defined(B)", |
| "(defined A)", {"B": "1"}) |
| test_cpp_expr_optim("defined(A) && defined(B)", "(int 0)", |
| {"B": kCppUndefinedMacro}) |
| test_cpp_expr_optim("defined(A) && defined(B)", |
| "(int 0)", {"A": kCppUndefinedMacro}) |
| test_cpp_expr_optim("A == 1 || defined(B)", |
| "(|| (== (ident A) (int 1)) (defined B))") |
| test_cpp_expr_optim( |
| "defined(__KERNEL__) || !defined(__GLIBC__) || (__GLIBC__ < 2)", |
| "(|| (! (defined __GLIBC__)) (< (ident __GLIBC__) (int 2)))", |
| {"__KERNEL__": kCppUndefinedMacro}) |
| |
| test_cpp_expr_source("0", "0") |
| test_cpp_expr_source("1", "1") |
| test_cpp_expr_source("1 && 1", "1 && 1") |
| test_cpp_expr_source("1 && 0", "1 && 0") |
| test_cpp_expr_source("0 && 1", "0 && 1") |
| test_cpp_expr_source("0 && 0", "0 && 0") |
| test_cpp_expr_source("1 || 1", "1 || 1") |
| test_cpp_expr_source("1 || 0", "1 || 0") |
| test_cpp_expr_source("0 || 1", "0 || 1") |
| test_cpp_expr_source("0 || 0", "0 || 0") |
| test_cpp_expr_source("EXAMPLE", "EXAMPLE") |
| test_cpp_expr_source("EXAMPLE - 3", "EXAMPLE - 3") |
| test_cpp_expr_source("defined(EXAMPLE)", "defined(EXAMPLE)") |
| test_cpp_expr_source("defined EXAMPLE", "defined(EXAMPLE)") |
| test_cpp_expr_source("A == 1 || defined(B)", "A == 1 || defined(B)") |
| |
| |
| ################################################################################ |
| ################################################################################ |
| ##### ##### |
| ##### C P P B L O C K ##### |
| ##### ##### |
| ################################################################################ |
| ################################################################################ |
| |
| |
| class Block(object): |
| """A class used to model a block of input source text. |
| |
| There are two block types: |
| - directive blocks: contain the tokens of a single pre-processor |
| directive (e.g. #if) |
| - text blocks, contain the tokens of non-directive blocks |
| |
| The cpp parser class below will transform an input source file into a list |
| of Block objects (grouped in a BlockList object for convenience) |
| """ |
| |
| def __init__(self, tokens, directive=None, lineno=0, identifier=None): |
| """Initialize a new block, if 'directive' is None, it is a text block. |
| |
| NOTE: This automatically converts '#ifdef MACRO' into |
| '#if defined(MACRO)' and '#ifndef MACRO' into '#if !defined(MACRO)'. |
| """ |
| |
| if directive == "ifdef": |
| tok = Token() |
| tok.id = tokDEFINED |
| tokens = [tok] + tokens |
| directive = "if" |
| |
| elif directive == "ifndef": |
| tok1 = Token() |
| tok2 = Token() |
| tok1.id = tokNOT |
| tok2.id = tokDEFINED |
| tokens = [tok1, tok2] + tokens |
| directive = "if" |
| |
| self.tokens = tokens |
| self.directive = directive |
| self.define_id = identifier |
| if lineno > 0: |
| self.lineno = lineno |
| else: |
| self.lineno = self.tokens[0].location.line |
| |
| if self.isIf(): |
| self.expr = CppExpr(self.tokens) |
| |
| def isDirective(self): |
| """Return True iff this is a directive block.""" |
| return self.directive is not None |
| |
| def isConditional(self): |
| """Return True iff this is a conditional directive block.""" |
| return self.directive in ["if", "ifdef", "ifndef", "else", "elif", |
| "endif"] |
| |
| def isDefine(self): |
| """Return the macro name in a #define directive, or None otherwise.""" |
| if self.directive != "define": |
| return None |
| return self.define_id |
| |
| def isIf(self): |
| """Return True iff this is an #if-like directive block.""" |
| return self.directive in ["if", "ifdef", "ifndef", "elif"] |
| |
| def isEndif(self): |
| """Return True iff this is an #endif directive block.""" |
| return self.directive == "endif" |
| |
| def isInclude(self): |
| """Check whether this is a #include directive. |
| |
| If true, returns the corresponding file name (with brackets or |
| double-qoutes). None otherwise. |
| """ |
| |
| if self.directive != "include": |
| return None |
| return ''.join([str(x) for x in self.tokens]) |
| |
| @staticmethod |
| def format_blocks(tokens, indent=0): |
| """Return the formatted lines of strings with proper indentation.""" |
| newline = True |
| result = [] |
| buf = '' |
| i = 0 |
| while i < len(tokens): |
| t = tokens[i] |
| if t.id == '{': |
| buf += ' {' |
| result.append(strip_space(buf)) |
| indent += 2 |
| buf = '' |
| newline = True |
| elif t.id == '}': |
| indent -= 2 |
| if not newline: |
| result.append(strip_space(buf)) |
| # Look ahead to determine if it's the end of line. |
| if (i + 1 < len(tokens) and |
| (tokens[i+1].id == ';' or |
| tokens[i+1].id in ['else', '__attribute__', |
| '__attribute', '__packed'] or |
| tokens[i+1].kind == TokenKind.IDENTIFIER)): |
| buf = ' ' * indent + '}' |
| newline = False |
| else: |
| result.append(' ' * indent + '}') |
| buf = '' |
| newline = True |
| elif t.id == ';': |
| result.append(strip_space(buf) + ';') |
| buf = '' |
| newline = True |
| # We prefer a new line for each constant in enum. |
| elif t.id == ',' and t.cursor.kind == CursorKind.ENUM_DECL: |
| result.append(strip_space(buf) + ',') |
| buf = '' |
| newline = True |
| else: |
| if newline: |
| buf += ' ' * indent + str(t) |
| else: |
| buf += ' ' + str(t) |
| newline = False |
| i += 1 |
| |
| if buf: |
| result.append(strip_space(buf)) |
| |
| return result, indent |
| |
| def writeWithWarning(self, out, warning, left_count, repeat_count, indent): |
| """Dump the current block with warnings.""" |
| # removeWhiteSpace() will sometimes creates non-directive blocks |
| # without any tokens. These come from blocks that only contained |
| # empty lines and spaces. They should not be printed in the final |
| # output, and then should not be counted for this operation. |
| # |
| if self.directive is None and not self.tokens: |
| return left_count, indent |
| |
| if self.directive: |
| out.write(str(self) + '\n') |
| left_count -= 1 |
| if left_count == 0: |
| out.write(warning) |
| left_count = repeat_count |
| |
| else: |
| lines, indent = self.format_blocks(self.tokens, indent) |
| for line in lines: |
| out.write(line + '\n') |
| left_count -= 1 |
| if left_count == 0: |
| out.write(warning) |
| left_count = repeat_count |
| |
| return left_count, indent |
| |
| def __repr__(self): |
| """Generate the representation of a given block.""" |
| if self.directive: |
| result = "#%s " % self.directive |
| if self.isIf(): |
| result += repr(self.expr) |
| else: |
| for tok in self.tokens: |
| result += repr(tok) |
| else: |
| result = "" |
| for tok in self.tokens: |
| result += repr(tok) |
| |
| return result |
| |
| def __str__(self): |
| """Generate the string representation of a given block.""" |
| if self.directive: |
| # "#if" |
| if self.directive == "if": |
| # small optimization to re-generate #ifdef and #ifndef |
| e = self.expr.expr |
| op = e[0] |
| if op == "defined": |
| result = "#ifdef %s" % e[1] |
| elif op == "!" and e[1][0] == "defined": |
| result = "#ifndef %s" % e[1][1] |
| else: |
| result = "#if " + str(self.expr) |
| |
| # "#define" |
| elif self.isDefine(): |
| result = "#%s %s" % (self.directive, self.define_id) |
| if self.tokens: |
| result += " " |
| expr = strip_space(' '.join([tok.id for tok in self.tokens])) |
| # remove the space between name and '(' in function call |
| result += re.sub(r'(\w+) \(', r'\1(', expr) |
| |
| # "#error" |
| # Concatenating tokens with a space separator, because they may |
| # not be quoted and broken into several tokens |
| elif self.directive == "error": |
| result = "#error %s" % ' '.join([tok.id for tok in self.tokens]) |
| |
| else: |
| result = "#%s" % self.directive |
| if self.tokens: |
| result += " " |
| result += ''.join([tok.id for tok in self.tokens]) |
| else: |
| lines, _ = self.format_blocks(self.tokens) |
| result = '\n'.join(lines) |
| |
| return result |
| |
| |
| class BlockList(object): |
| """A convenience class used to hold and process a list of blocks. |
| |
| It calls the cpp parser to get the blocks. |
| """ |
| |
| def __init__(self, blocks): |
| self.blocks = blocks |
| |
| def __len__(self): |
| return len(self.blocks) |
| |
| def __getitem__(self, n): |
| return self.blocks[n] |
| |
| def __repr__(self): |
| return repr(self.blocks) |
| |
| def __str__(self): |
| result = '\n'.join([str(b) for b in self.blocks]) |
| return result |
| |
| def dump(self): |
| """Dump all the blocks in current BlockList.""" |
| print '##### BEGIN #####' |
| for i, b in enumerate(self.blocks): |
| print '### BLOCK %d ###' % i |
| print b |
| print '##### END #####' |
| |
| def optimizeIf01(self): |
| """Remove the code between #if 0 .. #endif in a BlockList.""" |
| self.blocks = optimize_if01(self.blocks) |
| |
| def optimizeMacros(self, macros): |
| """Remove known defined and undefined macros from a BlockList.""" |
| for b in self.blocks: |
| if b.isIf(): |
| b.expr.optimize(macros) |
| |
| def removeMacroDefines(self, macros): |
| """Remove known macro definitions from a BlockList.""" |
| self.blocks = remove_macro_defines(self.blocks, macros) |
| |
| def optimizeAll(self, macros): |
| self.optimizeMacros(macros) |
| self.optimizeIf01() |
| return |
| |
| def findIncludes(self): |
| """Return the list of included files in a BlockList.""" |
| result = [] |
| for b in self.blocks: |
| i = b.isInclude() |
| if i: |
| result.append(i) |
| return result |
| |
| def write(self, out): |
| out.write(str(self)) |
| |
| def writeWithWarning(self, out, warning, repeat_count): |
| left_count = repeat_count |
| indent = 0 |
| for b in self.blocks: |
| left_count, indent = b.writeWithWarning(out, warning, left_count, |
| repeat_count, indent) |
| |
| def removeVarsAndFuncs(self, knownStatics=None): |
| """Remove variable and function declarations. |
| |
| All extern and static declarations corresponding to variable and |
| function declarations are removed. We only accept typedefs and |
| enum/structs/union declarations. |
| |
| However, we keep the definitions corresponding to the set of known |
| static inline functions in the set 'knownStatics', which is useful |
| for optimized byteorder swap functions and stuff like that. |
| """ |
| |
| # NOTE: It's also removing function-like macros, such as __SYSCALL(...) |
| # in uapi/asm-generic/unistd.h, or KEY_FIELD(...) in linux/bcache.h. |
| # It could be problematic when we have function-like macros but without |
| # '}' following them. It will skip all the tokens/blocks until seeing a |
| # '}' as the function end. Fortunately we don't have such cases in the |
| # current kernel headers. |
| |
| # state = 0 => normal (i.e. LN + spaces) |
| # state = 1 => typedef/struct encountered, ends with ";" |
| # state = 2 => var declaration encountered, ends with ";" |
| # state = 3 => func declaration encountered, ends with "}" |
| |
| if knownStatics is None: |
| knownStatics = set() |
| state = 0 |
| depth = 0 |
| blocks2 = [] |
| skipTokens = False |
| for b in self.blocks: |
| if b.isDirective(): |
| blocks2.append(b) |
| else: |
| n = len(b.tokens) |
| i = 0 |
| if skipTokens: |
| first = n |
| else: |
| first = 0 |
| while i < n: |
| tok = b.tokens[i] |
| tokid = tok.id |
| # If we are not looking for the start of a new |
| # type/var/func, then skip over tokens until |
| # we find our terminator, managing the depth of |
| # accolades as we go. |
| if state > 0: |
| terminator = False |
| if tokid == '{': |
| depth += 1 |
| elif tokid == '}': |
| if depth > 0: |
| depth -= 1 |
| if (depth == 0) and (state == 3): |
| terminator = True |
| elif tokid == ';' and depth == 0: |
| terminator = True |
| |
| if terminator: |
| # we found the terminator |
| state = 0 |
| if skipTokens: |
| skipTokens = False |
| first = i + 1 |
| |
| i += 1 |
| continue |
| |
| # Is it a new type definition, then start recording it |
| if tok.id in ['struct', 'typedef', 'enum', 'union', |
| '__extension__']: |
| state = 1 |
| i += 1 |
| continue |
| |
| # Is it a variable or function definition. If so, first |
| # try to determine which type it is, and also extract |
| # its name. |
| # |
| # We're going to parse the next tokens of the same block |
| # until we find a semi-column or a left parenthesis. |
| # |
| # The semi-column corresponds to a variable definition, |
| # the left-parenthesis to a function definition. |
| # |
| # We also assume that the var/func name is the last |
| # identifier before the terminator. |
| # |
| j = i + 1 |
| ident = "" |
| while j < n: |
| tokid = b.tokens[j].id |
| if tokid == '(': # a function declaration |
| state = 3 |
| break |
| elif tokid == ';': # a variable declaration |
| state = 2 |
| break |
| if b.tokens[j].kind == TokenKind.IDENTIFIER: |
| ident = b.tokens[j].id |
| j += 1 |
| |
| if j >= n: |
| # This can only happen when the declaration |
| # does not end on the current block (e.g. with |
| # a directive mixed inside it. |
| # |
| # We will treat it as malformed because |
| # it's very hard to recover from this case |
| # without making our parser much more |
| # complex. |
| # |
| logging.debug("### skip unterminated static '%s'", |
| ident) |
| break |
| |
| if ident in knownStatics: |
| logging.debug("### keep var/func '%s': %s", ident, |
| repr(b.tokens[i:j])) |
| else: |
| # We're going to skip the tokens for this declaration |
| logging.debug("### skip var/func '%s': %s", ident, |
| repr(b.tokens[i:j])) |
| if i > first: |
| blocks2.append(Block(b.tokens[first:i])) |
| skipTokens = True |
| first = n |
| |
| i += 1 |
| |
| if i > first: |
| # print "### final '%s'" % repr(b.tokens[first:i]) |
| blocks2.append(Block(b.tokens[first:i])) |
| |
| self.blocks = blocks2 |
| |
| def replaceTokens(self, replacements): |
| """Replace tokens according to the given dict.""" |
| for b in self.blocks: |
| made_change = False |
| if b.isInclude() is None: |
| for tok in b.tokens: |
| if tok.kind == TokenKind.IDENTIFIER: |
| if tok.id in replacements: |
| tok.id = replacements[tok.id] |
| made_change = True |
| |
| if b.isDefine() and b.define_id in replacements: |
| b.define_id = replacements[b.define_id] |
| made_change = True |
| |
| if made_change and b.isIf(): |
| # Keep 'expr' in sync with 'tokens'. |
| b.expr = CppExpr(b.tokens) |
| |
| |
| def strip_space(s): |
| """Strip out redundant space in a given string.""" |
| |
| # NOTE: It ought to be more clever to not destroy spaces in string tokens. |
| replacements = {' . ': '.', |
| ' [': '[', |
| '[ ': '[', |
| ' ]': ']', |
| '( ': '(', |
| ' )': ')', |
| ' ,': ',', |
| '# ': '#', |
| ' ;': ';', |
| '~ ': '~', |
| ' -> ': '->'} |
| result = s |
| for r in replacements: |
| result = result.replace(r, replacements[r]) |
| |
| # Remove the space between function name and the parenthesis. |
| result = re.sub(r'(\w+) \(', r'\1(', result) |
| return result |
| |
| |
| class BlockParser(object): |
| """A class that converts an input source file into a BlockList object.""" |
| |
| def __init__(self, tokzer=None): |
| """Initialize a block parser. |
| |
| The input source is provided through a Tokenizer object. |
| """ |
| self._tokzer = tokzer |
| self._parsed = False |
| |
| @property |
| def parsed(self): |
| return self._parsed |
| |
| @staticmethod |
| def _short_extent(extent): |
| return '%d:%d - %d:%d' % (extent.start.line, extent.start.column, |
| extent.end.line, extent.end.column) |
| |
| def getBlocks(self, tokzer=None): |
| """Return all the blocks parsed.""" |
| |
| def consume_extent(i, tokens, extent=None, detect_change=False): |
| """Return tokens that belong to the given extent. |
| |
| It parses all the tokens that follow tokens[i], until getting out |
| of the extent. When detect_change is True, it may terminate early |
| when detecting preprocessing directives inside the extent. |
| """ |
| |
| result = [] |
| if extent is None: |
| extent = tokens[i].cursor.extent |
| |
| while i < len(tokens) and tokens[i].location in extent: |
| t = tokens[i] |
| if debugBlockParser: |
| print ' ' * 2, t.id, t.kind, t.cursor.kind |
| if (detect_change and t.cursor.extent != extent and |
| t.cursor.kind == CursorKind.PREPROCESSING_DIRECTIVE): |
| break |
| result.append(t) |
| i += 1 |
| return (i, result) |
| |
| def consume_line(i, tokens): |
| """Return tokens that follow tokens[i] in the same line.""" |
| result = [] |
| line = tokens[i].location.line |
| while i < len(tokens) and tokens[i].location.line == line: |
| if tokens[i].cursor.kind == CursorKind.PREPROCESSING_DIRECTIVE: |
| break |
| result.append(tokens[i]) |
| i += 1 |
| return (i, result) |
| |
| if tokzer is None: |
| tokzer = self._tokzer |
| tokens = tokzer.tokens |
| |
| blocks = [] |
| buf = [] |
| i = 0 |
| |
| while i < len(tokens): |
| t = tokens[i] |
| cursor = t.cursor |
| |
| if debugBlockParser: |
| print ("%d: Processing [%s], kind=[%s], cursor=[%s], " |
| "extent=[%s]" % (t.location.line, t.spelling, t.kind, |
| cursor.kind, |
| self._short_extent(cursor.extent))) |
| |
| if cursor.kind == CursorKind.PREPROCESSING_DIRECTIVE: |
| if buf: |
| blocks.append(Block(buf)) |
| buf = [] |
| |
| j = i |
| if j + 1 >= len(tokens): |
| raise BadExpectedToken("### BAD TOKEN at %s" % (t.location)) |
| directive = tokens[j+1].id |
| |
| if directive == 'define': |
| if i+2 >= len(tokens): |
| raise BadExpectedToken("### BAD TOKEN at %s" % |
| (tokens[i].location)) |
| |
| # Skip '#' and 'define'. |
| extent = tokens[i].cursor.extent |
| i += 2 |
| id = '' |
| # We need to separate the id from the remaining of |
| # the line, especially for the function-like macro. |
| if (i + 1 < len(tokens) and tokens[i+1].id == '(' and |
| (tokens[i].location.column + len(tokens[i].spelling) == |
| tokens[i+1].location.column)): |
| while i < len(tokens): |
| id += tokens[i].id |
| if tokens[i].spelling == ')': |
| i += 1 |
| break |
| i += 1 |
| else: |
| id += tokens[i].id |
| # Advance to the next token that follows the macro id |
| i += 1 |
| |
| (i, ret) = consume_extent(i, tokens, extent=extent) |
| blocks.append(Block(ret, directive=directive, |
| lineno=t.location.line, identifier=id)) |
| |
| else: |
| (i, ret) = consume_extent(i, tokens) |
| blocks.append(Block(ret[2:], directive=directive, |
| lineno=t.location.line)) |
| |
| elif cursor.kind == CursorKind.INCLUSION_DIRECTIVE: |
| if buf: |
| blocks.append(Block(buf)) |
| buf = [] |
| directive = tokens[i+1].id |
| (i, ret) = consume_extent(i, tokens) |
| |
| blocks.append(Block(ret[2:], directive=directive, |
| lineno=t.location.line)) |
| |
| elif cursor.kind == CursorKind.VAR_DECL: |
| if buf: |
| blocks.append(Block(buf)) |
| buf = [] |
| |
| (i, ret) = consume_extent(i, tokens, detect_change=True) |
| buf += ret |
| |
| elif cursor.kind == CursorKind.FUNCTION_DECL: |
| if buf: |
| blocks.append(Block(buf)) |
| buf = [] |
| |
| (i, ret) = consume_extent(i, tokens, detect_change=True) |
| buf += ret |
| |
| else: |
| (i, ret) = consume_line(i, tokens) |
| buf += ret |
| |
| if buf: |
| blocks.append(Block(buf)) |
| |
| # _parsed=True indicates a successful parsing, although may result an |
| # empty BlockList. |
| self._parsed = True |
| |
| return BlockList(blocks) |
| |
| def parse(self, tokzer): |
| return self.getBlocks(tokzer) |
| |
| def parseFile(self, path): |
| return self.getBlocks(CppFileTokenizer(path)) |
| |
| |
| def test_block_parsing(lines, expected): |
| """Helper method to test the correctness of BlockParser.parse.""" |
| blocks = BlockParser().parse(CppStringTokenizer('\n'.join(lines))) |
| if len(blocks) != len(expected): |
| raise BadExpectedToken("BlockParser.parse() returned '%s' expecting " |
| "'%s'" % (str(blocks), repr(expected))) |
| for n in range(len(blocks)): |
| if str(blocks[n]) != expected[n]: |
| raise BadExpectedToken("BlockParser.parse()[%d] is '%s', " |
| "expecting '%s'" % (n, str(blocks[n]), |
| expected[n])) |
| |
| |
| def test_BlockParser(): |
| test_block_parsing(["#error hello"], ["#error hello"]) |
| test_block_parsing(["foo", "", "bar"], ["foo bar"]) |
| |
| # We currently cannot handle the following case with libclang properly. |
| # Fortunately it doesn't appear in current headers. |
| # test_block_parsing(["foo", " # ", "bar"], ["foo", "bar"]) |
| |
| test_block_parsing(["foo", |
| " # /* ahah */ if defined(__KERNEL__) /* more */", |
| "bar", "#endif"], |
| ["foo", "#ifdef __KERNEL__", "bar", "#endif"]) |
| |
| |
| ################################################################################ |
| ################################################################################ |
| ##### ##### |
| ##### B L O C K L I S T O P T I M I Z A T I O N ##### |
| ##### ##### |
| ################################################################################ |
| ################################################################################ |
| |
| |
| def remove_macro_defines(blocks, excludedMacros=None): |
| """Remove macro definitions like #define <macroName> ....""" |
| if excludedMacros is None: |
| excludedMacros = set() |
| result = [] |
| for b in blocks: |
| macroName = b.isDefine() |
| if macroName is None or macroName not in excludedMacros: |
| result.append(b) |
| |
| return result |
| |
| |
| def find_matching_endif(blocks, i): |
| """Traverse the blocks to find out the matching #endif.""" |
| n = len(blocks) |
| depth = 1 |
| while i < n: |
| if blocks[i].isDirective(): |
| dir_ = blocks[i].directive |
| if dir_ in ["if", "ifndef", "ifdef"]: |
| depth += 1 |
| elif depth == 1 and dir_ in ["else", "elif"]: |
| return i |
| elif dir_ == "endif": |
| depth -= 1 |
| if depth == 0: |
| return i |
| i += 1 |
| return i |
| |
| |
| def optimize_if01(blocks): |
| """Remove the code between #if 0 .. #endif in a list of CppBlocks.""" |
| i = 0 |
| n = len(blocks) |
| result = [] |
| while i < n: |
| j = i |
| while j < n and not blocks[j].isIf(): |
| j += 1 |
| if j > i: |
| logging.debug("appending lines %d to %d", blocks[i].lineno, |
| blocks[j-1].lineno) |
| result += blocks[i:j] |
| if j >= n: |
| break |
| expr = blocks[j].expr |
| r = expr.toInt() |
| if r is None: |
| result.append(blocks[j]) |
| i = j + 1 |
| continue |
| |
| if r == 0: |
| # if 0 => skip everything until the corresponding #endif |
| j = find_matching_endif(blocks, j + 1) |
| if j >= n: |
| # unterminated #if 0, finish here |
| break |
| dir_ = blocks[j].directive |
| if dir_ == "endif": |
| logging.debug("remove 'if 0' .. 'endif' (lines %d to %d)", |
| blocks[i].lineno, blocks[j].lineno) |
| i = j + 1 |
| elif dir_ == "else": |
| # convert 'else' into 'if 1' |
| logging.debug("convert 'if 0' .. 'else' into 'if 1' (lines %d " |
| "to %d)", blocks[i].lineno, blocks[j-1].lineno) |
| blocks[j].directive = "if" |
| blocks[j].expr = CppExpr(CppStringTokenizer("1").tokens) |
| i = j |
| elif dir_ == "elif": |
| # convert 'elif' into 'if' |
| logging.debug("convert 'if 0' .. 'elif' into 'if'") |
| blocks[j].directive = "if" |
| i = j |
| continue |
| |
| # if 1 => find corresponding endif and remove/transform them |
| k = find_matching_endif(blocks, j + 1) |
| if k >= n: |
| # unterminated #if 1, finish here |
| logging.debug("unterminated 'if 1'") |
| result += blocks[j+1:k] |
| break |
| |
| dir_ = blocks[k].directive |
| if dir_ == "endif": |
| logging.debug("convert 'if 1' .. 'endif' (lines %d to %d)", |
| blocks[j].lineno, blocks[k].lineno) |
| result += optimize_if01(blocks[j+1:k]) |
| i = k + 1 |
| elif dir_ == "else": |
| # convert 'else' into 'if 0' |
| logging.debug("convert 'if 1' .. 'else' (lines %d to %d)", |
| blocks[j].lineno, blocks[k].lineno) |
| result += optimize_if01(blocks[j+1:k]) |
| blocks[k].directive = "if" |
| blocks[k].expr = CppExpr(CppStringTokenizer("0").tokens) |
| i = k |
| elif dir_ == "elif": |
| # convert 'elif' into 'if 0' |
| logging.debug("convert 'if 1' .. 'elif' (lines %d to %d)", |
| blocks[j].lineno, blocks[k].lineno) |
| result += optimize_if01(blocks[j+1:k]) |
| blocks[k].expr = CppExpr(CppStringTokenizer("0").tokens) |
| i = k |
| return result |
| |
| |
| def test_optimizeAll(): |
| text = """\ |
| #if 1 |
| #define GOOD_1 |
| #endif |
| #if 0 |
| #define BAD_2 |
| #define BAD_3 |
| #endif |
| |
| #if 1 |
| #define GOOD_2 |
| #else |
| #define BAD_4 |
| #endif |
| |
| #if 0 |
| #define BAD_5 |
| #else |
| #define GOOD_3 |
| #endif |
| |
| #if defined(__KERNEL__) |
| #define BAD_KERNEL |
| #endif |
| |
| #if defined(__KERNEL__) || !defined(__GLIBC__) || (__GLIBC__ < 2) |
| #define X |
| #endif |
| |
| #ifndef SIGRTMAX |
| #define SIGRTMAX 123 |
| #endif /* SIGRTMAX */ |
| |
| #if 0 |
| #if 1 |
| #define BAD_6 |
| #endif |
| #endif\ |
| """ |
| |
| expected = """\ |
| #define GOOD_1 |
| #define GOOD_2 |
| #define GOOD_3 |
| #if !defined(__GLIBC__) || __GLIBC__ < 2 |
| #define X |
| #endif |
| #ifndef __SIGRTMAX |
| #define __SIGRTMAX 123 |
| #endif\ |
| """ |
| |
| out = utils.StringOutput() |
| blocks = BlockParser().parse(CppStringTokenizer(text)) |
| blocks.replaceTokens(kernel_token_replacements) |
| blocks.optimizeAll({"__KERNEL__": kCppUndefinedMacro}) |
| blocks.write(out) |
| if out.get() != expected: |
| print "[FAIL]: macro optimization failed\n" |
| print "<<<< expecting '", |
| print expected, |
| print "'\n>>>> result '", |
| print out.get(), |
| print "'\n----" |
| global failure_count |
| failure_count += 1 |
| |
| |
| def runUnitTests(): |
| """Always run all unit tests for this program.""" |
| test_CppTokenizer() |
| test_CppExpr() |
| test_optimizeAll() |
| test_BlockParser() |
| |
| |
| failure_count = 0 |
| runUnitTests() |
| if failure_count != 0: |
| utils.panic("Unit tests failed in cpp.py.\n") |