From ccad493e85e46bf0e78cfac8b77b03b7be75a396 Mon Sep 17 00:00:00 2001 From: Daniel Mensinger Date: Tue, 15 Jan 2019 18:45:25 +0100 Subject: Basic AST visitor pattern --- mesonbuild/ast/__init__.py | 25 ++++ mesonbuild/ast/interpreter.py | 332 ++++++++++++++++++++++++++++++++++++++++++ mesonbuild/ast/printer.py | 22 +++ mesonbuild/ast/visitor.py | 118 +++++++++++++++ mesonbuild/astinterpreter.py | 281 ----------------------------------- mesonbuild/mintro.py | 18 +-- mesonbuild/mparser.py | 50 ++++--- mesonbuild/rewriter.py | 26 ++-- 8 files changed, 539 insertions(+), 333 deletions(-) create mode 100644 mesonbuild/ast/__init__.py create mode 100644 mesonbuild/ast/interpreter.py create mode 100644 mesonbuild/ast/printer.py create mode 100644 mesonbuild/ast/visitor.py delete mode 100644 mesonbuild/astinterpreter.py diff --git a/mesonbuild/ast/__init__.py b/mesonbuild/ast/__init__.py new file mode 100644 index 000000000..4b82d7e68 --- /dev/null +++ b/mesonbuild/ast/__init__.py @@ -0,0 +1,25 @@ +# Copyright 2019 The Meson development team + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This class contains the basic functionality needed to run any interpreter +# or an interpreter-based tool. + +__all__ = [ + 'AstInterpreter', + 'RewriterInterpreter', + 'AstVisitor' +] + +from .interpreter import (AstInterpreter, RewriterInterpreter) +from .visitor import AstVisitor diff --git a/mesonbuild/ast/interpreter.py b/mesonbuild/ast/interpreter.py new file mode 100644 index 000000000..c1068d4fc --- /dev/null +++ b/mesonbuild/ast/interpreter.py @@ -0,0 +1,332 @@ +# Copyright 2016 The Meson development team + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This class contains the basic functionality needed to run any interpreter +# or an interpreter-based tool. + +from .. import interpreterbase, mparser, mesonlib +from .. import environment + +from ..interpreterbase import InterpreterException, InvalidArguments, BreakRequest, ContinueRequest + +import os, sys + +class DontCareObject(interpreterbase.InterpreterObject): + pass + +class MockExecutable(interpreterbase.InterpreterObject): + pass + +class MockStaticLibrary(interpreterbase.InterpreterObject): + pass + +class MockSharedLibrary(interpreterbase.InterpreterObject): + pass + +class MockCustomTarget(interpreterbase.InterpreterObject): + pass + +class MockRunTarget(interpreterbase.InterpreterObject): + pass + +ADD_SOURCE = 0 +REMOVE_SOURCE = 1 + +class AstInterpreter(interpreterbase.InterpreterBase): + def __init__(self, source_root, subdir): + super().__init__(source_root, subdir) + self.visited_subdirs = {} + self.funcs.update({'project': self.func_do_nothing, + 'test': self.func_do_nothing, + 'benchmark': self.func_do_nothing, + 'install_headers': self.func_do_nothing, + 'install_man': self.func_do_nothing, + 'install_data': self.func_do_nothing, + 'install_subdir': self.func_do_nothing, + 'configuration_data': self.func_do_nothing, + 'configure_file': self.func_do_nothing, + 'find_program': self.func_do_nothing, + 'include_directories': self.func_do_nothing, + 'add_global_arguments': self.func_do_nothing, + 'add_global_link_arguments': self.func_do_nothing, + 'add_project_arguments': self.func_do_nothing, + 'add_project_link_arguments': self.func_do_nothing, + 'message': self.func_do_nothing, + 'generator': self.func_do_nothing, + 'error': self.func_do_nothing, + 'run_command': self.func_do_nothing, + 'assert': self.func_do_nothing, + 'subproject': self.func_do_nothing, + 'dependency': self.func_do_nothing, + 'get_option': self.func_do_nothing, + 'join_paths': self.func_do_nothing, + 'environment': self.func_do_nothing, + 'import': self.func_do_nothing, + 'vcs_tag': self.func_do_nothing, + 'add_languages': self.func_do_nothing, + 'declare_dependency': self.func_do_nothing, + 'files': self.func_do_nothing, + 'executable': self.func_do_nothing, + 'static_library': self.func_do_nothing, + 'shared_library': self.func_do_nothing, + 'library': self.func_do_nothing, + 'build_target': self.func_do_nothing, + 'custom_target': self.func_do_nothing, + 'run_target': self.func_do_nothing, + 'subdir': self.func_subdir, + 'set_variable': self.func_do_nothing, + 'get_variable': self.func_do_nothing, + 'is_variable': self.func_do_nothing, + }) + + def func_do_nothing(self, node, args, kwargs): + return True + + def func_subdir(self, node, args, kwargs): + args = self.flatten_args(args) + if len(args) != 1 or not isinstance(args[0], str): + sys.stderr.write('Unable to evaluate subdir({}) in AstInterpreter --> Skipping\n'.format(args)) + return + + prev_subdir = self.subdir + subdir = os.path.join(prev_subdir, args[0]) + absdir = os.path.join(self.source_root, subdir) + buildfilename = os.path.join(self.subdir, environment.build_filename) + absname = os.path.join(self.source_root, buildfilename) + symlinkless_dir = os.path.realpath(absdir) + if symlinkless_dir in self.visited_subdirs: + sys.stderr.write('Trying to enter {} which has already been visited --> Skipping\n'.format(args[0])) + return + self.visited_subdirs[symlinkless_dir] = True + + if not os.path.isfile(absname): + sys.stderr.write('Unable to find build file {} --> Skipping\n'.format(buildfilename)) + return + with open(absname, encoding='utf8') as f: + code = f.read() + assert(isinstance(code, str)) + try: + codeblock = mparser.Parser(code, self.subdir).parse() + except mesonlib.MesonException as me: + me.file = buildfilename + raise me + + self.subdir = subdir + self.evaluate_codeblock(codeblock) + self.subdir = prev_subdir + + def method_call(self, node): + return True + + def evaluate_arithmeticstatement(self, cur): + return 0 + + def evaluate_plusassign(self, node): + return 0 + + def evaluate_indexing(self, node): + return 0 + + def unknown_function_called(self, func_name): + pass + + def reduce_arguments(self, args): + assert(isinstance(args, mparser.ArgumentNode)) + if args.incorrect_order(): + raise InvalidArguments('All keyword arguments must be after positional arguments.') + return args.arguments, args.kwargs + + def evaluate_comparison(self, node): + return False + + def evaluate_foreach(self, node): + try: + self.evaluate_codeblock(node.block) + except ContinueRequest: + pass + except BreakRequest: + pass + + def evaluate_if(self, node): + for i in node.ifs: + self.evaluate_codeblock(i.block) + if not isinstance(node.elseblock, mparser.EmptyNode): + self.evaluate_codeblock(node.elseblock) + + def get_variable(self, varname): + return 0 + + def assignment(self, node): + pass + + def flatten_args(self, args): + # Resolve mparser.ArrayNode if needed + flattend_args = [] + if isinstance(args, mparser.ArrayNode): + args = [x.value for x in args.args.arguments] + for i in args: + if isinstance(i, mparser.ArrayNode): + flattend_args += [x.value for x in i.args.arguments] + elif isinstance(i, str): + flattend_args += [i] + else: + pass + return flattend_args + +class RewriterInterpreter(AstInterpreter): + def __init__(self, source_root, subdir): + super().__init__(source_root, subdir) + self.asts = {} + self.funcs.update({'files': self.func_files, + 'executable': self.func_executable, + 'static_library': self.func_static_lib, + 'shared_library': self.func_shared_lib, + 'library': self.func_library, + 'build_target': self.func_build_target, + 'custom_target': self.func_custom_target, + 'run_target': self.func_run_target, + 'subdir': self.func_subdir, + 'set_variable': self.func_set_variable, + 'get_variable': self.func_get_variable, + 'is_variable': self.func_is_variable, + }) + + def func_executable(self, node, args, kwargs): + if args[0] == self.targetname: + if self.operation == ADD_SOURCE: + self.add_source_to_target(node, args, kwargs) + elif self.operation == REMOVE_SOURCE: + self.remove_source_from_target(node, args, kwargs) + else: + raise NotImplementedError('Bleep bloop') + return MockExecutable() + + def func_static_lib(self, node, args, kwargs): + return MockStaticLibrary() + + def func_shared_lib(self, node, args, kwargs): + return MockSharedLibrary() + + def func_library(self, node, args, kwargs): + return self.func_shared_lib(node, args, kwargs) + + def func_custom_target(self, node, args, kwargs): + return MockCustomTarget() + + def func_run_target(self, node, args, kwargs): + return MockRunTarget() + + def func_subdir(self, node, args, kwargs): + prev_subdir = self.subdir + subdir = os.path.join(prev_subdir, args[0]) + self.subdir = subdir + buildfilename = os.path.join(self.subdir, environment.build_filename) + absname = os.path.join(self.source_root, buildfilename) + if not os.path.isfile(absname): + self.subdir = prev_subdir + raise InterpreterException('Nonexistent build def file %s.' % buildfilename) + with open(absname, encoding='utf8') as f: + code = f.read() + assert(isinstance(code, str)) + try: + codeblock = mparser.Parser(code, self.subdir).parse() + self.asts[subdir] = codeblock + except mesonlib.MesonException as me: + me.file = buildfilename + raise me + try: + self.evaluate_codeblock(codeblock) + except SubdirDoneRequest: + pass + self.subdir = prev_subdir + + def func_files(self, node, args, kwargs): + if not isinstance(args, list): + return [args] + return args + + def transform(self): + self.load_root_meson_file() + self.asts[''] = self.ast + self.sanity_check_ast() + self.parse_project() + self.run() + + def add_source(self, targetname, filename): + self.operation = ADD_SOURCE + self.targetname = targetname + self.filename = filename + self.transform() + + def remove_source(self, targetname, filename): + self.operation = REMOVE_SOURCE + self.targetname = targetname + self.filename = filename + self.transform() + + def add_source_to_target(self, node, args, kwargs): + namespan = node.args.arguments[0].bytespan + buildfilename = os.path.join(self.source_root, self.subdir, environment.build_filename) + raw_data = open(buildfilename, 'r').read() + updated = raw_data[0:namespan[1]] + (", '%s'" % self.filename) + raw_data[namespan[1]:] + open(buildfilename, 'w').write(updated) + sys.exit(0) + + def remove_argument_item(self, args, i): + assert(isinstance(args, mparser.ArgumentNode)) + namespan = args.arguments[i].bytespan + # Usually remove the comma after this item but if it is + # the last argument, we need to remove the one before. + if i >= len(args.commas): + i -= 1 + if i < 0: + commaspan = (0, 0) # Removed every entry in the list. + else: + commaspan = args.commas[i].bytespan + if commaspan[0] < namespan[0]: + commaspan, namespan = namespan, commaspan + buildfilename = os.path.join(self.source_root, args.subdir, environment.build_filename) + raw_data = open(buildfilename, 'r').read() + intermediary = raw_data[0:commaspan[0]] + raw_data[commaspan[1]:] + updated = intermediary[0:namespan[0]] + intermediary[namespan[1]:] + open(buildfilename, 'w').write(updated) + sys.exit(0) + + def hacky_find_and_remove(self, node_to_remove): + for a in self.asts[node_to_remove.subdir].lines: + if a.lineno == node_to_remove.lineno: + if isinstance(a, mparser.AssignmentNode): + v = a.value + if not isinstance(v, mparser.ArrayNode): + raise NotImplementedError('Not supported yet, bro.') + args = v.args + for i in range(len(args.arguments)): + if isinstance(args.arguments[i], mparser.StringNode) and self.filename == args.arguments[i].value: + self.remove_argument_item(args, i) + raise NotImplementedError('Sukkess') + + def remove_source_from_target(self, node, args, kwargs): + for i in range(1, len(node.args)): + # Is file name directly in function call as a string. + if isinstance(node.args.arguments[i], mparser.StringNode) and self.filename == node.args.arguments[i].value: + self.remove_argument_item(node.args, i) + # Is file name in a variable that gets expanded here. + if isinstance(node.args.arguments[i], mparser.IdNode): + avar = self.get_variable(node.args.arguments[i].value) + if not isinstance(avar, list): + raise NotImplementedError('Non-arrays not supported yet, sorry.') + for entry in avar: + if isinstance(entry, mparser.StringNode) and entry.value == self.filename: + self.hacky_find_and_remove(entry) + sys.exit('Could not find source %s in target %s.' % (self.filename, args[0])) diff --git a/mesonbuild/ast/printer.py b/mesonbuild/ast/printer.py new file mode 100644 index 000000000..c1710bec7 --- /dev/null +++ b/mesonbuild/ast/printer.py @@ -0,0 +1,22 @@ +# Copyright 2018 The Meson development team + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This class contains the basic functionality needed to run any interpreter +# or an interpreter-based tool + +from . import AstVisitor + +class AstPrinter(AstVisitor): + def __init__(self): + pass diff --git a/mesonbuild/ast/visitor.py b/mesonbuild/ast/visitor.py new file mode 100644 index 000000000..487cd5bcc --- /dev/null +++ b/mesonbuild/ast/visitor.py @@ -0,0 +1,118 @@ +# Copyright 2018 The Meson development team + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This class contains the basic functionality needed to run any interpreter +# or an interpreter-based tool + +from .. import mparser + +class AstVisitor: + def __init__(self): + pass + + def visit_BooleanNode(self, node: mparser.BooleanNode): + pass + + def visit_IdNode(self, node: mparser.IdNode): + pass + + def visit_NumberNode(self, node: mparser.NumberNode): + pass + + def visit_StringNode(self, node: mparser.StringNode): + pass + + def visit_ContinueNode(self, node: mparser.ContinueNode): + pass + + def visit_BreakNode(self, node: mparser.BreakNode): + pass + + def visit_ArrayNode(self, node: mparser.ArrayNode): + node.args.accept(self) + + def visit_DictNode(self, node: mparser.DictNode): + node.args.accept(self) + + def visit_EmptyNode(self, node: mparser.EmptyNode): + pass + + def visit_OrNode(self, node: mparser.OrNode): + node.left.accept(self) + node.right.accept(self) + + def visit_AndNode(self, node: mparser.AndNode): + node.left.accept(self) + node.right.accept(self) + + def visit_ComparisonNode(self, node: mparser.ComparisonNode): + node.left.accept(self) + node.right.accept(self) + + def visit_ArithmeticNode(self, node: mparser.ArithmeticNode): + node.left.accept(self) + node.right.accept(self) + + def visit_NotNode(self, node: mparser.NotNode): + node.value.accept(self) + + def visit_CodeBlockNode(self, node: mparser.CodeBlockNode): + for i in node.lines: + i.accept(self) + + def visit_IndexNode(self, node: mparser.IndexNode): + node.index.accept(self) + + def visit_MethodNode(self, node: mparser.MethodNode): + node.source_object.accept(self) + node.args.accept(self) + + def visit_FunctionNode(self, node: mparser.FunctionNode): + node.args.accept(self) + + def visit_AssignmentNode(self, node: mparser.AssignmentNode): + node.value.accept(self) + + def visit_PlusAssignmentNode(self, node: mparser.PlusAssignmentNode): + node.value.accept(self) + + def visit_ForeachClauseNode(self, node: mparser.ForeachClauseNode): + node.items.accept(self) + node.block.accept(self) + + def visit_IfClauseNode(self, node: mparser.IfClauseNode): + for i in node.ifs: + i.accept(self) + if node.elseblock: + node.elseblock.accept(self) + + def visit_UMinusNode(self, node: mparser.UMinusNode): + node.value.accept(self) + + def visit_IfNode(self, node: mparser.IfNode): + node.condition.accept(self) + node.block.accept(self) + + def visit_TernaryNode(self, node: mparser.TernaryNode): + node.condition.accept(self) + node.trueblock.accept(self) + node.falseblock.accept(self) + + def visit_ArgumentNode(self, node: mparser.ArgumentNode): + for i in node.arguments: + i.accept(self) + for i in node.commas: + pass + for val in node.kwargs.values(): + val.accept(self) diff --git a/mesonbuild/astinterpreter.py b/mesonbuild/astinterpreter.py deleted file mode 100644 index f68aa7a14..000000000 --- a/mesonbuild/astinterpreter.py +++ /dev/null @@ -1,281 +0,0 @@ -# Copyright 2016 The Meson development team - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This class contains the basic functionality needed to run any interpreter -# or an interpreter-based tool. - -from . import interpreterbase, mparser, mesonlib -from . import environment - -from .interpreterbase import InterpreterException, InvalidArguments, BreakRequest, ContinueRequest - -import os, sys - -class DontCareObject(interpreterbase.InterpreterObject): - pass - -class MockExecutable(interpreterbase.InterpreterObject): - pass - -class MockStaticLibrary(interpreterbase.InterpreterObject): - pass - -class MockSharedLibrary(interpreterbase.InterpreterObject): - pass - -class MockCustomTarget(interpreterbase.InterpreterObject): - pass - -class MockRunTarget(interpreterbase.InterpreterObject): - pass - -ADD_SOURCE = 0 -REMOVE_SOURCE = 1 - -class AstInterpreter(interpreterbase.InterpreterBase): - def __init__(self, source_root, subdir): - super().__init__(source_root, subdir) - self.funcs.update({'project': self.func_do_nothing, - 'test': self.func_do_nothing, - 'benchmark': self.func_do_nothing, - 'install_headers': self.func_do_nothing, - 'install_man': self.func_do_nothing, - 'install_data': self.func_do_nothing, - 'install_subdir': self.func_do_nothing, - 'configuration_data': self.func_do_nothing, - 'configure_file': self.func_do_nothing, - 'find_program': self.func_do_nothing, - 'include_directories': self.func_do_nothing, - 'add_global_arguments': self.func_do_nothing, - 'add_global_link_arguments': self.func_do_nothing, - 'add_project_arguments': self.func_do_nothing, - 'add_project_link_arguments': self.func_do_nothing, - 'message': self.func_do_nothing, - 'generator': self.func_do_nothing, - 'error': self.func_do_nothing, - 'run_command': self.func_do_nothing, - 'assert': self.func_do_nothing, - 'subproject': self.func_do_nothing, - 'dependency': self.func_do_nothing, - 'get_option': self.func_do_nothing, - 'join_paths': self.func_do_nothing, - 'environment': self.func_do_nothing, - 'import': self.func_do_nothing, - 'vcs_tag': self.func_do_nothing, - 'add_languages': self.func_do_nothing, - 'declare_dependency': self.func_do_nothing, - 'files': self.func_do_nothing, - 'executable': self.func_do_nothing, - 'static_library': self.func_do_nothing, - 'shared_library': self.func_do_nothing, - 'library': self.func_do_nothing, - 'build_target': self.func_do_nothing, - 'custom_target': self.func_do_nothing, - 'run_target': self.func_do_nothing, - 'subdir': self.func_do_nothing, - 'set_variable': self.func_do_nothing, - 'get_variable': self.func_do_nothing, - 'is_variable': self.func_do_nothing, - }) - - def func_do_nothing(self, node, args, kwargs): - return True - - def method_call(self, node): - return True - - def evaluate_arithmeticstatement(self, cur): - return 0 - - def evaluate_plusassign(self, node): - return 0 - - def evaluate_indexing(self, node): - return 0 - - def unknown_function_called(self, func_name): - pass - - def reduce_arguments(self, args): - assert(isinstance(args, mparser.ArgumentNode)) - if args.incorrect_order(): - raise InvalidArguments('All keyword arguments must be after positional arguments.') - return args.arguments, args.kwargs - - def evaluate_comparison(self, node): - return False - - def evaluate_foreach(self, node): - try: - self.evaluate_codeblock(node.block) - except ContinueRequest: - pass - except BreakRequest: - pass - - def evaluate_if(self, node): - for i in node.ifs: - self.evaluate_codeblock(i.block) - if not isinstance(node.elseblock, mparser.EmptyNode): - self.evaluate_codeblock(node.elseblock) - - def get_variable(self, varname): - return 0 - - def assignment(self, node): - pass - -class RewriterInterpreter(AstInterpreter): - def __init__(self, source_root, subdir): - super().__init__(source_root, subdir) - self.asts = {} - self.funcs.update({'files': self.func_files, - 'executable': self.func_executable, - 'static_library': self.func_static_lib, - 'shared_library': self.func_shared_lib, - 'library': self.func_library, - 'build_target': self.func_build_target, - 'custom_target': self.func_custom_target, - 'run_target': self.func_run_target, - 'subdir': self.func_subdir, - 'set_variable': self.func_set_variable, - 'get_variable': self.func_get_variable, - 'is_variable': self.func_is_variable, - }) - - def func_executable(self, node, args, kwargs): - if args[0] == self.targetname: - if self.operation == ADD_SOURCE: - self.add_source_to_target(node, args, kwargs) - elif self.operation == REMOVE_SOURCE: - self.remove_source_from_target(node, args, kwargs) - else: - raise NotImplementedError('Bleep bloop') - return MockExecutable() - - def func_static_lib(self, node, args, kwargs): - return MockStaticLibrary() - - def func_shared_lib(self, node, args, kwargs): - return MockSharedLibrary() - - def func_library(self, node, args, kwargs): - return self.func_shared_lib(node, args, kwargs) - - def func_custom_target(self, node, args, kwargs): - return MockCustomTarget() - - def func_run_target(self, node, args, kwargs): - return MockRunTarget() - - def func_subdir(self, node, args, kwargs): - prev_subdir = self.subdir - subdir = os.path.join(prev_subdir, args[0]) - self.subdir = subdir - buildfilename = os.path.join(self.subdir, environment.build_filename) - absname = os.path.join(self.source_root, buildfilename) - if not os.path.isfile(absname): - self.subdir = prev_subdir - raise InterpreterException('Nonexistent build def file %s.' % buildfilename) - with open(absname, encoding='utf8') as f: - code = f.read() - assert(isinstance(code, str)) - try: - codeblock = mparser.Parser(code, self.subdir).parse() - self.asts[subdir] = codeblock - except mesonlib.MesonException as me: - me.file = buildfilename - raise me - self.evaluate_codeblock(codeblock) - self.subdir = prev_subdir - - def func_files(self, node, args, kwargs): - if not isinstance(args, list): - return [args] - return args - - def transform(self): - self.load_root_meson_file() - self.asts[''] = self.ast - self.sanity_check_ast() - self.parse_project() - self.run() - - def add_source(self, targetname, filename): - self.operation = ADD_SOURCE - self.targetname = targetname - self.filename = filename - self.transform() - - def remove_source(self, targetname, filename): - self.operation = REMOVE_SOURCE - self.targetname = targetname - self.filename = filename - self.transform() - - def add_source_to_target(self, node, args, kwargs): - namespan = node.args.arguments[0].bytespan - buildfilename = os.path.join(self.source_root, self.subdir, environment.build_filename) - raw_data = open(buildfilename, 'r').read() - updated = raw_data[0:namespan[1]] + (", '%s'" % self.filename) + raw_data[namespan[1]:] - open(buildfilename, 'w').write(updated) - sys.exit(0) - - def remove_argument_item(self, args, i): - assert(isinstance(args, mparser.ArgumentNode)) - namespan = args.arguments[i].bytespan - # Usually remove the comma after this item but if it is - # the last argument, we need to remove the one before. - if i >= len(args.commas): - i -= 1 - if i < 0: - commaspan = (0, 0) # Removed every entry in the list. - else: - commaspan = args.commas[i].bytespan - if commaspan[0] < namespan[0]: - commaspan, namespan = namespan, commaspan - buildfilename = os.path.join(self.source_root, args.subdir, environment.build_filename) - raw_data = open(buildfilename, 'r').read() - intermediary = raw_data[0:commaspan[0]] + raw_data[commaspan[1]:] - updated = intermediary[0:namespan[0]] + intermediary[namespan[1]:] - open(buildfilename, 'w').write(updated) - sys.exit(0) - - def hacky_find_and_remove(self, node_to_remove): - for a in self.asts[node_to_remove.subdir].lines: - if a.lineno == node_to_remove.lineno: - if isinstance(a, mparser.AssignmentNode): - v = a.value - if not isinstance(v, mparser.ArrayNode): - raise NotImplementedError('Not supported yet, bro.') - args = v.args - for i in range(len(args.arguments)): - if isinstance(args.arguments[i], mparser.StringNode) and self.filename == args.arguments[i].value: - self.remove_argument_item(args, i) - raise NotImplementedError('Sukkess') - - def remove_source_from_target(self, node, args, kwargs): - for i in range(1, len(node.args)): - # Is file name directly in function call as a string. - if isinstance(node.args.arguments[i], mparser.StringNode) and self.filename == node.args.arguments[i].value: - self.remove_argument_item(node.args, i) - # Is file name in a variable that gets expanded here. - if isinstance(node.args.arguments[i], mparser.IdNode): - avar = self.get_variable(node.args.arguments[i].value) - if not isinstance(avar, list): - raise NotImplementedError('Non-arrays not supported yet, sorry.') - for entry in avar: - if isinstance(entry, mparser.StringNode) and entry.value == self.filename: - self.hacky_find_and_remove(entry) - sys.exit('Could not find source %s in target %s.' % (self.filename, args[0])) diff --git a/mesonbuild/mintro.py b/mesonbuild/mintro.py index 36368af99..fca238ffe 100644 --- a/mesonbuild/mintro.py +++ b/mesonbuild/mintro.py @@ -23,7 +23,7 @@ import json from . import build, coredata as cdata from . import environment from . import mesonlib -from . import astinterpreter +from .ast import AstInterpreter from . import mparser from . import mlog from . import compilers @@ -158,7 +158,7 @@ class IntrospectionHelper: self.native_file = None self.cmd_line_options = {} -class IntrospectionInterpreter(astinterpreter.AstInterpreter): +class IntrospectionInterpreter(AstInterpreter): # Interpreter to detect the options without a build directory # Most of the code is stolen from interperter.Interpreter def __init__(self, source_root, subdir, backend, cross_file=None, subproject='', subproject_dir='subprojects', env=None): @@ -183,20 +183,6 @@ class IntrospectionInterpreter(astinterpreter.AstInterpreter): 'add_languages': self.func_add_languages }) - def flatten_args(self, args): - # Resolve mparser.ArrayNode if needed - flattend_args = [] - if isinstance(args, mparser.ArrayNode): - args = [x.value for x in args.args.arguments] - for i in args: - if isinstance(i, mparser.ArrayNode): - flattend_args += [x.value for x in i.args.arguments] - elif isinstance(i, str): - flattend_args += [i] - else: - pass - return flattend_args - def func_project(self, node, args, kwargs): if len(args) < 1: raise InvalidArguments('Not enough arguments to project(). Needs at least the project name.') diff --git a/mesonbuild/mparser.py b/mesonbuild/mparser.py index fd8052e87..845a1a115 100644 --- a/mesonbuild/mparser.py +++ b/mesonbuild/mparser.py @@ -212,7 +212,15 @@ This will become a hard error in a future Meson release.""", self.getline(line_s if not matched: raise ParseException('lexer', self.getline(line_start), lineno, col) -class ElementaryNode: +class BaseNode: + def accept(self, visitor): + fname = 'visit_{}'.format(type(self).__name__) + if hasattr(visitor, fname): + func = getattr(visitor, fname) + if hasattr(func, '__call__'): + func(self) + +class ElementaryNode(BaseNode): def __init__(self, token): self.lineno = token.lineno self.subdir = token.subdir @@ -253,28 +261,28 @@ class ContinueNode(ElementaryNode): class BreakNode(ElementaryNode): pass -class ArrayNode: +class ArrayNode(BaseNode): def __init__(self, args): self.subdir = args.subdir self.lineno = args.lineno self.colno = args.colno self.args = args -class DictNode: +class DictNode(BaseNode): def __init__(self, args): self.subdir = args.subdir self.lineno = args.lineno self.colno = args.colno self.args = args -class EmptyNode: +class EmptyNode(BaseNode): def __init__(self, lineno, colno): self.subdir = '' self.lineno = lineno self.colno = colno self.value = None -class OrNode: +class OrNode(BaseNode): def __init__(self, left, right): self.subdir = left.subdir self.lineno = left.lineno @@ -282,7 +290,7 @@ class OrNode: self.left = left self.right = right -class AndNode: +class AndNode(BaseNode): def __init__(self, left, right): self.subdir = left.subdir self.lineno = left.lineno @@ -290,7 +298,7 @@ class AndNode: self.left = left self.right = right -class ComparisonNode: +class ComparisonNode(BaseNode): def __init__(self, ctype, left, right): self.lineno = left.lineno self.colno = left.colno @@ -299,7 +307,7 @@ class ComparisonNode: self.right = right self.ctype = ctype -class ArithmeticNode: +class ArithmeticNode(BaseNode): def __init__(self, operation, left, right): self.subdir = left.subdir self.lineno = left.lineno @@ -308,21 +316,21 @@ class ArithmeticNode: self.right = right self.operation = operation -class NotNode: +class NotNode(BaseNode): def __init__(self, location_node, value): self.subdir = location_node.subdir self.lineno = location_node.lineno self.colno = location_node.colno self.value = value -class CodeBlockNode: +class CodeBlockNode(BaseNode): def __init__(self, location_node): self.subdir = location_node.subdir self.lineno = location_node.lineno self.colno = location_node.colno self.lines = [] -class IndexNode: +class IndexNode(BaseNode): def __init__(self, iobject, index): self.iobject = iobject self.index = index @@ -330,7 +338,7 @@ class IndexNode: self.lineno = iobject.lineno self.colno = iobject.colno -class MethodNode: +class MethodNode(BaseNode): def __init__(self, subdir, lineno, colno, source_object, name, args): self.subdir = subdir self.lineno = lineno @@ -340,7 +348,7 @@ class MethodNode: assert(isinstance(self.name, str)) self.args = args -class FunctionNode: +class FunctionNode(BaseNode): def __init__(self, subdir, lineno, colno, func_name, args): self.subdir = subdir self.lineno = lineno @@ -349,7 +357,7 @@ class FunctionNode: assert(isinstance(func_name, str)) self.args = args -class AssignmentNode: +class AssignmentNode(BaseNode): def __init__(self, lineno, colno, var_name, value): self.lineno = lineno self.colno = colno @@ -357,7 +365,7 @@ class AssignmentNode: assert(isinstance(var_name, str)) self.value = value -class PlusAssignmentNode: +class PlusAssignmentNode(BaseNode): def __init__(self, lineno, colno, var_name, value): self.lineno = lineno self.colno = colno @@ -365,7 +373,7 @@ class PlusAssignmentNode: assert(isinstance(var_name, str)) self.value = value -class ForeachClauseNode: +class ForeachClauseNode(BaseNode): def __init__(self, lineno, colno, varnames, items, block): self.lineno = lineno self.colno = colno @@ -373,28 +381,28 @@ class ForeachClauseNode: self.items = items self.block = block -class IfClauseNode: +class IfClauseNode(BaseNode): def __init__(self, lineno, colno): self.lineno = lineno self.colno = colno self.ifs = [] self.elseblock = EmptyNode(lineno, colno) -class UMinusNode: +class UMinusNode(BaseNode): def __init__(self, current_location, value): self.subdir = current_location.subdir self.lineno = current_location.lineno self.colno = current_location.colno self.value = value -class IfNode: +class IfNode(BaseNode): def __init__(self, lineno, colno, condition, block): self.lineno = lineno self.colno = colno self.condition = condition self.block = block -class TernaryNode: +class TernaryNode(BaseNode): def __init__(self, lineno, colno, condition, trueblock, falseblock): self.lineno = lineno self.colno = colno @@ -402,7 +410,7 @@ class TernaryNode: self.trueblock = trueblock self.falseblock = falseblock -class ArgumentNode: +class ArgumentNode(BaseNode): def __init__(self, token): self.lineno = token.lineno self.colno = token.colno diff --git a/mesonbuild/rewriter.py b/mesonbuild/rewriter.py index 37ed7efd6..1495d2314 100644 --- a/mesonbuild/rewriter.py +++ b/mesonbuild/rewriter.py @@ -23,7 +23,7 @@ # - move targets # - reindent? -import mesonbuild.astinterpreter +from .ast import (AstInterpreter, AstVisitor) from mesonbuild.mesonlib import MesonException from mesonbuild import mlog import sys, traceback @@ -31,24 +31,20 @@ import sys, traceback def add_arguments(parser): parser.add_argument('--sourcedir', default='.', help='Path to source directory.') - parser.add_argument('--target', default=None, - help='Name of target to edit.') - parser.add_argument('--filename', default=None, - help='Name of source file to add or remove to target.') - parser.add_argument('commands', nargs='+') + parser.add_argument('-p', '--print', action='store_true', default=False, dest='print', + help='Print the parsed AST.') def run(options): - if options.target is None or options.filename is None: - sys.exit("Must specify both target and filename.") print('This tool is highly experimental, use with care.') - rewriter = mesonbuild.astinterpreter.RewriterInterpreter(options.sourcedir, '') + rewriter = AstInterpreter(options.sourcedir, '') try: - if options.commands[0] == 'add': - rewriter.add_source(options.target, options.filename) - elif options.commands[0] == 'remove': - rewriter.remove_source(options.target, options.filename) - else: - sys.exit('Unknown command: ' + options.commands[0]) + rewriter.load_root_meson_file() + rewriter.sanity_check_ast() + rewriter.parse_project() + rewriter.run() + + visitor = AstVisitor() + rewriter.ast.accept(visitor) except Exception as e: if isinstance(e, MesonException): mlog.exception(e) -- cgit v1.2.3 From 46320bfba89bf088dfe760ee1ff39658b5eb559f Mon Sep 17 00:00:00 2001 From: Daniel Mensinger Date: Tue, 15 Jan 2019 21:59:49 +0100 Subject: Added Ast printer --- mesonbuild/ast/__init__.py | 4 +- mesonbuild/ast/printer.py | 192 ++++++++++++++++++++++++++++++++++++++++++++- mesonbuild/rewriter.py | 6 +- 3 files changed, 196 insertions(+), 6 deletions(-) diff --git a/mesonbuild/ast/__init__.py b/mesonbuild/ast/__init__.py index 4b82d7e68..1469cf5da 100644 --- a/mesonbuild/ast/__init__.py +++ b/mesonbuild/ast/__init__.py @@ -18,8 +18,10 @@ __all__ = [ 'AstInterpreter', 'RewriterInterpreter', - 'AstVisitor' + 'AstVisitor', + 'AstPrinter' ] from .interpreter import (AstInterpreter, RewriterInterpreter) from .visitor import AstVisitor +from .printer import AstPrinter diff --git a/mesonbuild/ast/printer.py b/mesonbuild/ast/printer.py index c1710bec7..b0554802f 100644 --- a/mesonbuild/ast/printer.py +++ b/mesonbuild/ast/printer.py @@ -15,8 +15,196 @@ # This class contains the basic functionality needed to run any interpreter # or an interpreter-based tool +from .. import mparser from . import AstVisitor +arithmic_map = { + 'add': '+', + 'sub': '-', + 'mod': '%', + 'mul': '*', + 'div': '/' +} + class AstPrinter(AstVisitor): - def __init__(self): - pass + def __init__(self, indent: int = 2, arg_newline_cutoff: int = 5): + self.result = '' + self.indent = indent + self.arg_newline_cutoff = arg_newline_cutoff + self.level = 0 + self.ci = '' + self.is_newline = True + + def inc_indent(self): + self.level += self.indent + + def dec_indent(self): + self.level -= self.indent + + def append(self, data: str): + if self.is_newline: + self.result += ' ' * self.level + self.result += data + self.is_newline = False + + def appendS(self, data: str): + if self.result[-1] not in [' ', '\n']: + data = ' ' + data + self.append(data + ' ') + + def newline(self): + self.result += '\n' + self.is_newline = True + + def visit_BooleanNode(self, node: mparser.BooleanNode): + self.append('true' if node.value else 'false') + + def visit_IdNode(self, node: mparser.IdNode): + self.append(node.value) + + def visit_NumberNode(self, node: mparser.NumberNode): + self.append(str(node.value)) + + def visit_StringNode(self, node: mparser.StringNode): + self.append("'" + node.value + "'") + + def visit_ContinueNode(self, node: mparser.ContinueNode): + self.append('continue') + + def visit_BreakNode(self, node: mparser.BreakNode): + self.append('break') + + def visit_ArrayNode(self, node: mparser.ArrayNode): + self.append('[') + self.inc_indent() + node.args.accept(self) + self.dec_indent() + self.append(']') + + def visit_DictNode(self, node: mparser.DictNode): + self.append('{') + self.inc_indent() + node.args.accept(self) + self.dec_indent() + self.append('}') + + def visit_OrNode(self, node: mparser.OrNode): + node.left.accept(self) + self.appendS('or') + node.right.accept(self) + + def visit_AndNode(self, node: mparser.AndNode): + node.left.accept(self) + self.appendS('and') + node.right.accept(self) + + def visit_ComparisonNode(self, node: mparser.ComparisonNode): + node.left.accept(self) + self.appendS(mparser.comparison_map[node.ctype]) + node.right.accept(self) + + def visit_ArithmeticNode(self, node: mparser.ArithmeticNode): + node.left.accept(self) + self.appendS(arithmic_map[node.operation]) + node.right.accept(self) + + def visit_NotNode(self, node: mparser.NotNode): + self.appendS('not') + node.value.accept(self) + + def visit_CodeBlockNode(self, node: mparser.CodeBlockNode): + for i in node.lines: + i.accept(self) + self.newline() + + def visit_IndexNode(self, node: mparser.IndexNode): + self.append('[') + node.index.accept(self) + self.append(']') + + def visit_MethodNode(self, node: mparser.MethodNode): + node.source_object.accept(self) + self.append('.' + node.name + '(') + self.inc_indent() + node.args.accept(self) + self.dec_indent() + self.append(')') + + def visit_FunctionNode(self, node: mparser.FunctionNode): + self.append(node.func_name + '(') + self.inc_indent() + node.args.accept(self) + self.dec_indent() + self.append(')') + + def visit_AssignmentNode(self, node: mparser.AssignmentNode): + self.append(node.var_name + ' = ') + node.value.accept(self) + + def visit_PlusAssignmentNode(self, node: mparser.PlusAssignmentNode): + self.append(node.var_name + ' += ') + node.value.accept(self) + + def visit_ForeachClauseNode(self, node: mparser.ForeachClauseNode): + varnames = [x.value for x in node.varnames] + self.appendS('foreach') + self.appendS(', '.join(varnames)) + self.appendS(':') + node.items.accept(self) + self.newline() + self.inc_indent() + node.block.accept(self) + self.dec_indent() + self.append('endforeach') + + def visit_IfClauseNode(self, node: mparser.IfClauseNode): + prefix = '' + for i in node.ifs: + self.appendS(prefix + 'if') + prefix = 'el' + i.accept(self) + if node.elseblock: + self.append('else') + self.indent() + self.inc_indent() + node.elseblock.accept(self) + self.dec_indent() + self.append('endif') + + def visit_UMinusNode(self, node: mparser.UMinusNode): + self.appendS('-') + node.value.accept(self) + + def visit_IfNode(self, node: mparser.IfNode): + node.condition.accept(self) + self.newline() + self.inc_indent() + node.block.accept(self) + self.dec_indent() + + def visit_TernaryNode(self, node: mparser.TernaryNode): + node.condition.accept(self) + self.appendS('?') + node.trueblock.accept(self) + self.appendS(':') + node.falseblock.accept(self) + + def visit_ArgumentNode(self, node: mparser.ArgumentNode): + break_args = True if (len(node.arguments) + len(node.kwargs)) > self.arg_newline_cutoff else False + for i in node.arguments + list(node.kwargs.values()): + if not isinstance(i, mparser.ElementaryNode): + break_args = True + if break_args: + self.newline() + for i in node.arguments: + i.accept(self) + self.append(',') + if break_args: + self.newline() + for key, val in node.kwargs.items(): + self.append(key) + self.appendS(':') + val.accept(self) + self.append(',') + if break_args: + self.newline() diff --git a/mesonbuild/rewriter.py b/mesonbuild/rewriter.py index 1495d2314..f025f23b0 100644 --- a/mesonbuild/rewriter.py +++ b/mesonbuild/rewriter.py @@ -23,7 +23,7 @@ # - move targets # - reindent? -from .ast import (AstInterpreter, AstVisitor) +from .ast import (AstInterpreter, AstVisitor, AstPrinter) from mesonbuild.mesonlib import MesonException from mesonbuild import mlog import sys, traceback @@ -35,7 +35,6 @@ def add_arguments(parser): help='Print the parsed AST.') def run(options): - print('This tool is highly experimental, use with care.') rewriter = AstInterpreter(options.sourcedir, '') try: rewriter.load_root_meson_file() @@ -43,8 +42,9 @@ def run(options): rewriter.parse_project() rewriter.run() - visitor = AstVisitor() + visitor = AstPrinter() rewriter.ast.accept(visitor) + print(visitor.result) except Exception as e: if isinstance(e, MesonException): mlog.exception(e) -- cgit v1.2.3 From 750af9c853423138e0d2812ba538402eedea80f2 Mon Sep 17 00:00:00 2001 From: Daniel Mensinger Date: Sat, 19 Jan 2019 14:03:34 +0100 Subject: Moved the introspection interpreter --- mesonbuild/ast/__init__.py | 4 +- mesonbuild/ast/interpreter.py | 5 +- mesonbuild/ast/introspection.py | 125 ++++++++++++++++++++++++++++++++++++++++ mesonbuild/mintro.py | 109 +---------------------------------- mesonbuild/rewriter.py | 4 +- 5 files changed, 132 insertions(+), 115 deletions(-) create mode 100644 mesonbuild/ast/introspection.py diff --git a/mesonbuild/ast/__init__.py b/mesonbuild/ast/__init__.py index 1469cf5da..181c5493c 100644 --- a/mesonbuild/ast/__init__.py +++ b/mesonbuild/ast/__init__.py @@ -19,9 +19,11 @@ __all__ = [ 'AstInterpreter', 'RewriterInterpreter', 'AstVisitor', - 'AstPrinter' + 'AstPrinter', + 'IntrospectionInterpreter', ] from .interpreter import (AstInterpreter, RewriterInterpreter) +from .introspection import IntrospectionInterpreter from .visitor import AstVisitor from .printer import AstPrinter diff --git a/mesonbuild/ast/interpreter.py b/mesonbuild/ast/interpreter.py index c1068d4fc..b401aa826 100644 --- a/mesonbuild/ast/interpreter.py +++ b/mesonbuild/ast/interpreter.py @@ -245,10 +245,7 @@ class RewriterInterpreter(AstInterpreter): except mesonlib.MesonException as me: me.file = buildfilename raise me - try: - self.evaluate_codeblock(codeblock) - except SubdirDoneRequest: - pass + self.evaluate_codeblock(codeblock) self.subdir = prev_subdir def func_files(self, node, args, kwargs): diff --git a/mesonbuild/ast/introspection.py b/mesonbuild/ast/introspection.py new file mode 100644 index 000000000..f21788c44 --- /dev/null +++ b/mesonbuild/ast/introspection.py @@ -0,0 +1,125 @@ +# Copyright 2018 The Meson development team + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This class contains the basic functionality needed to run any interpreter +# or an interpreter-based tool + +from . import AstInterpreter +from .. import compilers, environment, mesonlib, mparser, optinterpreter +from .. import coredata as cdata +from ..interpreterbase import InvalidArguments + +import os + +class IntrospectionHelper: + # mimic an argparse namespace + def __init__(self, cross_file): + self.cross_file = cross_file + self.native_file = None + self.cmd_line_options = {} + +class IntrospectionInterpreter(AstInterpreter): + # Interpreter to detect the options without a build directory + # Most of the code is stolen from interperter.Interpreter + def __init__(self, source_root, subdir, backend, cross_file=None, subproject='', subproject_dir='subprojects', env=None): + super().__init__(source_root, subdir) + + options = IntrospectionHelper(cross_file) + self.cross_file = cross_file + if env is None: + self.environment = environment.Environment(source_root, None, options) + else: + self.environment = env + self.subproject = subproject + self.subproject_dir = subproject_dir + self.coredata = self.environment.get_coredata() + self.option_file = os.path.join(self.source_root, self.subdir, 'meson_options.txt') + self.backend = backend + self.default_options = {'backend': self.backend} + self.project_data = {} + + self.funcs.update({ + 'project': self.func_project, + 'add_languages': self.func_add_languages + }) + + def func_project(self, node, args, kwargs): + if len(args) < 1: + raise InvalidArguments('Not enough arguments to project(). Needs at least the project name.') + + proj_name = args[0] + proj_vers = kwargs.get('version', 'undefined') + proj_langs = self.flatten_args(args[1:]) + if isinstance(proj_vers, mparser.ElementaryNode): + proj_vers = proj_vers.value + if not isinstance(proj_vers, str): + proj_vers = 'undefined' + self.project_data = {'descriptive_name': proj_name, 'version': proj_vers} + + if os.path.exists(self.option_file): + oi = optinterpreter.OptionInterpreter(self.subproject) + oi.process(self.option_file) + self.coredata.merge_user_options(oi.options) + + def_opts = self.flatten_args(kwargs.get('default_options', [])) + self.project_default_options = mesonlib.stringlistify(def_opts) + self.project_default_options = cdata.create_options_dict(self.project_default_options) + self.default_options.update(self.project_default_options) + self.coredata.set_default_options(self.default_options, self.subproject, self.environment.cmd_line_options) + + if not self.is_subproject() and 'subproject_dir' in kwargs: + spdirname = kwargs['subproject_dir'] + if isinstance(spdirname, str): + self.subproject_dir = spdirname + if not self.is_subproject(): + self.project_data['subprojects'] = [] + subprojects_dir = os.path.join(self.source_root, self.subproject_dir) + if os.path.isdir(subprojects_dir): + for i in os.listdir(subprojects_dir): + if os.path.isdir(os.path.join(subprojects_dir, i)): + self.do_subproject(i) + + self.coredata.init_backend_options(self.backend) + options = {k: v for k, v in self.environment.cmd_line_options.items() if k.startswith('backend_')} + + self.coredata.set_options(options) + self.func_add_languages(None, proj_langs, None) + + def do_subproject(self, dirname): + subproject_dir_abs = os.path.join(self.environment.get_source_dir(), self.subproject_dir) + subpr = os.path.join(subproject_dir_abs, dirname) + try: + subi = IntrospectionInterpreter(subpr, '', self.backend, cross_file=self.cross_file, subproject=dirname, subproject_dir=self.subproject_dir, env=self.environment) + subi.analyze() + subi.project_data['name'] = dirname + self.project_data['subprojects'] += [subi.project_data] + except: + return + + def func_add_languages(self, node, args, kwargs): + args = self.flatten_args(args) + need_cross_compiler = self.environment.is_cross_build() + for lang in sorted(args, key=compilers.sort_clink): + lang = lang.lower() + if lang not in self.coredata.compilers: + self.environment.detect_compilers(lang, need_cross_compiler) + + def is_subproject(self): + return self.subproject != '' + + def analyze(self): + self.load_root_meson_file() + self.sanity_check_ast() + self.parse_project() + self.run() diff --git a/mesonbuild/mintro.py b/mesonbuild/mintro.py index fca238ffe..074c70a7d 100644 --- a/mesonbuild/mintro.py +++ b/mesonbuild/mintro.py @@ -21,14 +21,9 @@ project files and don't need this info.""" import json from . import build, coredata as cdata -from . import environment from . import mesonlib -from .ast import AstInterpreter -from . import mparser +from .ast import IntrospectionInterpreter from . import mlog -from . import compilers -from . import optinterpreter -from .interpreterbase import InvalidArguments from .backend import backends import sys, os import pathlib @@ -151,108 +146,6 @@ def list_targets(builddata: build.Build, installdata, backend: backends.Backend) tlist.append(t) return tlist -class IntrospectionHelper: - # mimic an argparse namespace - def __init__(self, cross_file): - self.cross_file = cross_file - self.native_file = None - self.cmd_line_options = {} - -class IntrospectionInterpreter(AstInterpreter): - # Interpreter to detect the options without a build directory - # Most of the code is stolen from interperter.Interpreter - def __init__(self, source_root, subdir, backend, cross_file=None, subproject='', subproject_dir='subprojects', env=None): - super().__init__(source_root, subdir) - - options = IntrospectionHelper(cross_file) - self.cross_file = cross_file - if env is None: - self.environment = environment.Environment(source_root, None, options) - else: - self.environment = env - self.subproject = subproject - self.subproject_dir = subproject_dir - self.coredata = self.environment.get_coredata() - self.option_file = os.path.join(self.source_root, self.subdir, 'meson_options.txt') - self.backend = backend - self.default_options = {'backend': self.backend} - self.project_data = {} - - self.funcs.update({ - 'project': self.func_project, - 'add_languages': self.func_add_languages - }) - - def func_project(self, node, args, kwargs): - if len(args) < 1: - raise InvalidArguments('Not enough arguments to project(). Needs at least the project name.') - - proj_name = args[0] - proj_vers = kwargs.get('version', 'undefined') - proj_langs = self.flatten_args(args[1:]) - if isinstance(proj_vers, mparser.ElementaryNode): - proj_vers = proj_vers.value - if not isinstance(proj_vers, str): - proj_vers = 'undefined' - self.project_data = {'descriptive_name': proj_name, 'version': proj_vers} - - if os.path.exists(self.option_file): - oi = optinterpreter.OptionInterpreter(self.subproject) - oi.process(self.option_file) - self.coredata.merge_user_options(oi.options) - - def_opts = self.flatten_args(kwargs.get('default_options', [])) - self.project_default_options = mesonlib.stringlistify(def_opts) - self.project_default_options = cdata.create_options_dict(self.project_default_options) - self.default_options.update(self.project_default_options) - self.coredata.set_default_options(self.default_options, self.subproject, self.environment.cmd_line_options) - - if not self.is_subproject() and 'subproject_dir' in kwargs: - spdirname = kwargs['subproject_dir'] - if isinstance(spdirname, str): - self.subproject_dir = spdirname - if not self.is_subproject(): - self.project_data['subprojects'] = [] - subprojects_dir = os.path.join(self.source_root, self.subproject_dir) - if os.path.isdir(subprojects_dir): - for i in os.listdir(subprojects_dir): - if os.path.isdir(os.path.join(subprojects_dir, i)): - self.do_subproject(i) - - self.coredata.init_backend_options(self.backend) - options = {k: v for k, v in self.environment.cmd_line_options.items() if k.startswith('backend_')} - - self.coredata.set_options(options) - self.func_add_languages(None, proj_langs, None) - - def do_subproject(self, dirname): - subproject_dir_abs = os.path.join(self.environment.get_source_dir(), self.subproject_dir) - subpr = os.path.join(subproject_dir_abs, dirname) - try: - subi = IntrospectionInterpreter(subpr, '', self.backend, cross_file=self.cross_file, subproject=dirname, subproject_dir=self.subproject_dir, env=self.environment) - subi.analyze() - subi.project_data['name'] = dirname - self.project_data['subprojects'] += [subi.project_data] - except: - return - - def func_add_languages(self, node, args, kwargs): - args = self.flatten_args(args) - need_cross_compiler = self.environment.is_cross_build() - for lang in sorted(args, key=compilers.sort_clink): - lang = lang.lower() - if lang not in self.coredata.compilers: - self.environment.detect_compilers(lang, need_cross_compiler) - - def is_subproject(self): - return self.subproject != '' - - def analyze(self): - self.load_root_meson_file() - self.sanity_check_ast() - self.parse_project() - self.run() - def list_buildoptions_from_source(sourcedir, backend, indent): # Make sure that log entries in other parts of meson don't interfere with the JSON output mlog.disable() diff --git a/mesonbuild/rewriter.py b/mesonbuild/rewriter.py index f025f23b0..45504be97 100644 --- a/mesonbuild/rewriter.py +++ b/mesonbuild/rewriter.py @@ -23,10 +23,10 @@ # - move targets # - reindent? -from .ast import (AstInterpreter, AstVisitor, AstPrinter) +from .ast import (AstInterpreter, AstPrinter) from mesonbuild.mesonlib import MesonException from mesonbuild import mlog -import sys, traceback +import traceback def add_arguments(parser): parser.add_argument('--sourcedir', default='.', -- cgit v1.2.3 From 277dc10a5d21eaed884ef8674ca6cc1bec547ec0 Mon Sep 17 00:00:00 2001 From: Daniel Mensinger Date: Sun, 20 Jan 2019 17:43:09 +0100 Subject: AST post processing --- mesonbuild/ast/__init__.py | 3 ++ mesonbuild/ast/postprocess.py | 86 +++++++++++++++++++++++++++++++++ mesonbuild/ast/printer.py | 109 +++++++++++++++++------------------------- mesonbuild/ast/visitor.py | 38 +++++++++++---- mesonbuild/rewriter.py | 12 +++-- run_unittests.py | 2 +- 6 files changed, 173 insertions(+), 77 deletions(-) create mode 100644 mesonbuild/ast/postprocess.py diff --git a/mesonbuild/ast/__init__.py b/mesonbuild/ast/__init__.py index 181c5493c..8d0a6ec5f 100644 --- a/mesonbuild/ast/__init__.py +++ b/mesonbuild/ast/__init__.py @@ -18,6 +18,8 @@ __all__ = [ 'AstInterpreter', 'RewriterInterpreter', + 'AstIDGenerator', + 'AstIndentationGenerator', 'AstVisitor', 'AstPrinter', 'IntrospectionInterpreter', @@ -26,4 +28,5 @@ __all__ = [ from .interpreter import (AstInterpreter, RewriterInterpreter) from .introspection import IntrospectionInterpreter from .visitor import AstVisitor +from .postprocess import AstIDGenerator, AstIndentationGenerator from .printer import AstPrinter diff --git a/mesonbuild/ast/postprocess.py b/mesonbuild/ast/postprocess.py new file mode 100644 index 000000000..e913b4f04 --- /dev/null +++ b/mesonbuild/ast/postprocess.py @@ -0,0 +1,86 @@ +# Copyright 2019 The Meson development team + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This class contains the basic functionality needed to run any interpreter +# or an interpreter-based tool + +from . import AstVisitor +from .. import mparser + +class AstIndentationGenerator(AstVisitor): + def __init__(self): + self.level = 0 + + def visit_default_func(self, node: mparser.BaseNode): + # Store the current level in the node + node.level = self.level + + def visit_ArrayNode(self, node: mparser.ArrayNode): + self.visit_default_func(node) + self.level += 1 + node.args.accept(self) + self.level -= 1 + + def visit_DictNode(self, node: mparser.DictNode): + self.visit_default_func(node) + self.level += 1 + node.args.accept(self) + self.level -= 1 + + def visit_MethodNode(self, node: mparser.MethodNode): + self.visit_default_func(node) + node.source_object.accept(self) + self.level += 1 + node.args.accept(self) + self.level -= 1 + + def visit_FunctionNode(self, node: mparser.FunctionNode): + self.visit_default_func(node) + self.level += 1 + node.args.accept(self) + self.level -= 1 + + def visit_ForeachClauseNode(self, node: mparser.ForeachClauseNode): + self.visit_default_func(node) + self.level += 1 + node.items.accept(self) + node.block.accept(self) + self.level -= 1 + + def visit_IfClauseNode(self, node: mparser.IfClauseNode): + self.visit_default_func(node) + for i in node.ifs: + i.accept(self) + if node.elseblock: + self.level += 1 + node.elseblock.accept(self) + self.level -= 1 + + def visit_IfNode(self, node: mparser.IfNode): + self.visit_default_func(node) + self.level += 1 + node.condition.accept(self) + node.block.accept(self) + self.level -= 1 + +class AstIDGenerator(AstVisitor): + def __init__(self): + self.counter = {} + + def visit_default_func(self, node: mparser.BaseNode): + name = type(node).__name__ + if name not in self.counter: + self.counter[name] = 0 + node.ast_id = name + '#' + str(self.counter[name]) + self.counter[name] += 1 diff --git a/mesonbuild/ast/printer.py b/mesonbuild/ast/printer.py index b0554802f..aab5a301f 100644 --- a/mesonbuild/ast/printer.py +++ b/mesonbuild/ast/printer.py @@ -1,4 +1,4 @@ -# Copyright 2018 The Meson development team +# Copyright 2019 The Meson development team # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -31,85 +31,77 @@ class AstPrinter(AstVisitor): self.result = '' self.indent = indent self.arg_newline_cutoff = arg_newline_cutoff - self.level = 0 self.ci = '' self.is_newline = True - def inc_indent(self): - self.level += self.indent - - def dec_indent(self): - self.level -= self.indent - - def append(self, data: str): + def append(self, data: str, node: mparser.BaseNode): + level = 0 + if node and hasattr(node, 'level'): + level = node.level if self.is_newline: - self.result += ' ' * self.level + self.result += ' ' * (level * self.indent) self.result += data self.is_newline = False - def appendS(self, data: str): + def appendS(self, data: str, node: mparser.BaseNode): if self.result[-1] not in [' ', '\n']: data = ' ' + data - self.append(data + ' ') + self.append(data + ' ', node) def newline(self): self.result += '\n' self.is_newline = True def visit_BooleanNode(self, node: mparser.BooleanNode): - self.append('true' if node.value else 'false') + self.append('true' if node.value else 'false', node) def visit_IdNode(self, node: mparser.IdNode): - self.append(node.value) + self.append(node.value, node) def visit_NumberNode(self, node: mparser.NumberNode): - self.append(str(node.value)) + self.append(str(node.value), node) def visit_StringNode(self, node: mparser.StringNode): - self.append("'" + node.value + "'") + self.append("'" + node.value + "'", node) def visit_ContinueNode(self, node: mparser.ContinueNode): - self.append('continue') + self.append('continue', node) def visit_BreakNode(self, node: mparser.BreakNode): - self.append('break') + self.append('break', node) def visit_ArrayNode(self, node: mparser.ArrayNode): - self.append('[') - self.inc_indent() + self.append('[', node) node.args.accept(self) - self.dec_indent() - self.append(']') + self.append(']', node) def visit_DictNode(self, node: mparser.DictNode): - self.append('{') - self.inc_indent() + self.append('{', node) node.args.accept(self) - self.dec_indent() - self.append('}') + self.append('}', node) def visit_OrNode(self, node: mparser.OrNode): node.left.accept(self) - self.appendS('or') + self.appendS('or', node) node.right.accept(self) def visit_AndNode(self, node: mparser.AndNode): node.left.accept(self) - self.appendS('and') + self.appendS('and', node) node.right.accept(self) def visit_ComparisonNode(self, node: mparser.ComparisonNode): node.left.accept(self) - self.appendS(mparser.comparison_map[node.ctype]) + self.appendS(mparser.comparison_map[node.ctype], node) node.right.accept(self) def visit_ArithmeticNode(self, node: mparser.ArithmeticNode): node.left.accept(self) - self.appendS(arithmic_map[node.operation]) + self.appendS(arithmic_map[node.operation], node) node.right.accept(self) def visit_NotNode(self, node: mparser.NotNode): - self.appendS('not') + self.appendS('not', node) node.value.accept(self) def visit_CodeBlockNode(self, node: mparser.CodeBlockNode): @@ -118,75 +110,64 @@ class AstPrinter(AstVisitor): self.newline() def visit_IndexNode(self, node: mparser.IndexNode): - self.append('[') + self.append('[', node) node.index.accept(self) - self.append(']') + self.append(']', node) def visit_MethodNode(self, node: mparser.MethodNode): node.source_object.accept(self) - self.append('.' + node.name + '(') - self.inc_indent() + self.append('.' + node.name + '(', node) node.args.accept(self) - self.dec_indent() - self.append(')') + self.append(')', node) def visit_FunctionNode(self, node: mparser.FunctionNode): - self.append(node.func_name + '(') - self.inc_indent() + self.append(node.func_name + '(', node) node.args.accept(self) - self.dec_indent() - self.append(')') + self.append(')', node) def visit_AssignmentNode(self, node: mparser.AssignmentNode): - self.append(node.var_name + ' = ') + self.append(node.var_name + ' = ', node) node.value.accept(self) def visit_PlusAssignmentNode(self, node: mparser.PlusAssignmentNode): - self.append(node.var_name + ' += ') + self.append(node.var_name + ' += ', node) node.value.accept(self) def visit_ForeachClauseNode(self, node: mparser.ForeachClauseNode): varnames = [x.value for x in node.varnames] - self.appendS('foreach') - self.appendS(', '.join(varnames)) - self.appendS(':') + self.appendS('foreach', node) + self.appendS(', '.join(varnames), node) + self.appendS(':', node) node.items.accept(self) self.newline() - self.inc_indent() node.block.accept(self) - self.dec_indent() - self.append('endforeach') + self.append('endforeach', node) def visit_IfClauseNode(self, node: mparser.IfClauseNode): prefix = '' for i in node.ifs: - self.appendS(prefix + 'if') + self.appendS(prefix + 'if', node) prefix = 'el' i.accept(self) if node.elseblock: - self.append('else') - self.indent() - self.inc_indent() + self.append('else', node) node.elseblock.accept(self) - self.dec_indent() - self.append('endif') + self.append('endif', node) def visit_UMinusNode(self, node: mparser.UMinusNode): - self.appendS('-') + self.appendS('-', node) node.value.accept(self) def visit_IfNode(self, node: mparser.IfNode): node.condition.accept(self) self.newline() - self.inc_indent() node.block.accept(self) - self.dec_indent() def visit_TernaryNode(self, node: mparser.TernaryNode): node.condition.accept(self) - self.appendS('?') + self.appendS('?', node) node.trueblock.accept(self) - self.appendS(':') + self.appendS(':', node) node.falseblock.accept(self) def visit_ArgumentNode(self, node: mparser.ArgumentNode): @@ -198,13 +179,13 @@ class AstPrinter(AstVisitor): self.newline() for i in node.arguments: i.accept(self) - self.append(',') + self.append(',', node) if break_args: self.newline() for key, val in node.kwargs.items(): - self.append(key) - self.appendS(':') + self.append(key, node) + self.appendS(':', node) val.accept(self) - self.append(',') + self.append(',', node) if break_args: self.newline() diff --git a/mesonbuild/ast/visitor.py b/mesonbuild/ast/visitor.py index 487cd5bcc..c8769d436 100644 --- a/mesonbuild/ast/visitor.py +++ b/mesonbuild/ast/visitor.py @@ -1,4 +1,4 @@ -# Copyright 2018 The Meson development team +# Copyright 2019 The Meson development team # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -21,95 +21,117 @@ class AstVisitor: def __init__(self): pass - def visit_BooleanNode(self, node: mparser.BooleanNode): + def visit_default_func(self, node: mparser.BaseNode): pass + def visit_BooleanNode(self, node: mparser.BooleanNode): + self.visit_default_func(node) + def visit_IdNode(self, node: mparser.IdNode): - pass + self.visit_default_func(node) def visit_NumberNode(self, node: mparser.NumberNode): - pass + self.visit_default_func(node) def visit_StringNode(self, node: mparser.StringNode): - pass + self.visit_default_func(node) def visit_ContinueNode(self, node: mparser.ContinueNode): - pass + self.visit_default_func(node) def visit_BreakNode(self, node: mparser.BreakNode): - pass + self.visit_default_func(node) def visit_ArrayNode(self, node: mparser.ArrayNode): + self.visit_default_func(node) node.args.accept(self) def visit_DictNode(self, node: mparser.DictNode): + self.visit_default_func(node) node.args.accept(self) def visit_EmptyNode(self, node: mparser.EmptyNode): - pass + self.visit_default_func(node) def visit_OrNode(self, node: mparser.OrNode): + self.visit_default_func(node) node.left.accept(self) node.right.accept(self) def visit_AndNode(self, node: mparser.AndNode): + self.visit_default_func(node) node.left.accept(self) node.right.accept(self) def visit_ComparisonNode(self, node: mparser.ComparisonNode): + self.visit_default_func(node) node.left.accept(self) node.right.accept(self) def visit_ArithmeticNode(self, node: mparser.ArithmeticNode): + self.visit_default_func(node) node.left.accept(self) node.right.accept(self) def visit_NotNode(self, node: mparser.NotNode): + self.visit_default_func(node) node.value.accept(self) def visit_CodeBlockNode(self, node: mparser.CodeBlockNode): + self.visit_default_func(node) for i in node.lines: i.accept(self) def visit_IndexNode(self, node: mparser.IndexNode): + self.visit_default_func(node) node.index.accept(self) def visit_MethodNode(self, node: mparser.MethodNode): + self.visit_default_func(node) node.source_object.accept(self) node.args.accept(self) def visit_FunctionNode(self, node: mparser.FunctionNode): + self.visit_default_func(node) node.args.accept(self) def visit_AssignmentNode(self, node: mparser.AssignmentNode): + self.visit_default_func(node) node.value.accept(self) def visit_PlusAssignmentNode(self, node: mparser.PlusAssignmentNode): + self.visit_default_func(node) node.value.accept(self) def visit_ForeachClauseNode(self, node: mparser.ForeachClauseNode): + self.visit_default_func(node) node.items.accept(self) node.block.accept(self) def visit_IfClauseNode(self, node: mparser.IfClauseNode): + self.visit_default_func(node) for i in node.ifs: i.accept(self) if node.elseblock: node.elseblock.accept(self) def visit_UMinusNode(self, node: mparser.UMinusNode): + self.visit_default_func(node) node.value.accept(self) def visit_IfNode(self, node: mparser.IfNode): + self.visit_default_func(node) node.condition.accept(self) node.block.accept(self) def visit_TernaryNode(self, node: mparser.TernaryNode): + self.visit_default_func(node) node.condition.accept(self) node.trueblock.accept(self) node.falseblock.accept(self) def visit_ArgumentNode(self, node: mparser.ArgumentNode): + self.visit_default_func(node) for i in node.arguments: i.accept(self) for i in node.commas: diff --git a/mesonbuild/rewriter.py b/mesonbuild/rewriter.py index 45504be97..cc5d2abef 100644 --- a/mesonbuild/rewriter.py +++ b/mesonbuild/rewriter.py @@ -23,7 +23,7 @@ # - move targets # - reindent? -from .ast import (AstInterpreter, AstPrinter) +from .ast import AstInterpreter, AstVisitor, AstIDGenerator, AstIndentationGenerator, AstPrinter from mesonbuild.mesonlib import MesonException from mesonbuild import mlog import traceback @@ -42,9 +42,13 @@ def run(options): rewriter.parse_project() rewriter.run() - visitor = AstPrinter() - rewriter.ast.accept(visitor) - print(visitor.result) + indentor = AstIndentationGenerator() + idgen = AstIDGenerator() + printer = AstPrinter() + rewriter.ast.accept(indentor) + rewriter.ast.accept(idgen) + rewriter.ast.accept(printer) + print(printer.result) except Exception as e: if isinstance(e, MesonException): mlog.exception(e) diff --git a/run_unittests.py b/run_unittests.py index e1246140a..8c2ad12f2 100755 --- a/run_unittests.py +++ b/run_unittests.py @@ -4913,7 +4913,7 @@ class PythonTests(BasePlatformTests): self.wipe() -class RewriterTests(unittest.TestCase): +class RewriterTests(BasePlatformTests): def setUp(self): super().setUp() -- cgit v1.2.3 From 005a62491b7f93c08dc471ee16d8f9e3b1888f55 Mon Sep 17 00:00:00 2001 From: Daniel Mensinger Date: Sun, 20 Jan 2019 17:51:52 +0100 Subject: Removed the RewriteIntrepreter --- mesonbuild/ast/__init__.py | 3 +- mesonbuild/ast/interpreter.py | 144 ------------------------------------------ 2 files changed, 1 insertion(+), 146 deletions(-) diff --git a/mesonbuild/ast/__init__.py b/mesonbuild/ast/__init__.py index 8d0a6ec5f..0d1a4d69d 100644 --- a/mesonbuild/ast/__init__.py +++ b/mesonbuild/ast/__init__.py @@ -17,7 +17,6 @@ __all__ = [ 'AstInterpreter', - 'RewriterInterpreter', 'AstIDGenerator', 'AstIndentationGenerator', 'AstVisitor', @@ -25,7 +24,7 @@ __all__ = [ 'IntrospectionInterpreter', ] -from .interpreter import (AstInterpreter, RewriterInterpreter) +from .interpreter import AstInterpreter from .introspection import IntrospectionInterpreter from .visitor import AstVisitor from .postprocess import AstIDGenerator, AstIndentationGenerator diff --git a/mesonbuild/ast/interpreter.py b/mesonbuild/ast/interpreter.py index b401aa826..032337833 100644 --- a/mesonbuild/ast/interpreter.py +++ b/mesonbuild/ast/interpreter.py @@ -183,147 +183,3 @@ class AstInterpreter(interpreterbase.InterpreterBase): else: pass return flattend_args - -class RewriterInterpreter(AstInterpreter): - def __init__(self, source_root, subdir): - super().__init__(source_root, subdir) - self.asts = {} - self.funcs.update({'files': self.func_files, - 'executable': self.func_executable, - 'static_library': self.func_static_lib, - 'shared_library': self.func_shared_lib, - 'library': self.func_library, - 'build_target': self.func_build_target, - 'custom_target': self.func_custom_target, - 'run_target': self.func_run_target, - 'subdir': self.func_subdir, - 'set_variable': self.func_set_variable, - 'get_variable': self.func_get_variable, - 'is_variable': self.func_is_variable, - }) - - def func_executable(self, node, args, kwargs): - if args[0] == self.targetname: - if self.operation == ADD_SOURCE: - self.add_source_to_target(node, args, kwargs) - elif self.operation == REMOVE_SOURCE: - self.remove_source_from_target(node, args, kwargs) - else: - raise NotImplementedError('Bleep bloop') - return MockExecutable() - - def func_static_lib(self, node, args, kwargs): - return MockStaticLibrary() - - def func_shared_lib(self, node, args, kwargs): - return MockSharedLibrary() - - def func_library(self, node, args, kwargs): - return self.func_shared_lib(node, args, kwargs) - - def func_custom_target(self, node, args, kwargs): - return MockCustomTarget() - - def func_run_target(self, node, args, kwargs): - return MockRunTarget() - - def func_subdir(self, node, args, kwargs): - prev_subdir = self.subdir - subdir = os.path.join(prev_subdir, args[0]) - self.subdir = subdir - buildfilename = os.path.join(self.subdir, environment.build_filename) - absname = os.path.join(self.source_root, buildfilename) - if not os.path.isfile(absname): - self.subdir = prev_subdir - raise InterpreterException('Nonexistent build def file %s.' % buildfilename) - with open(absname, encoding='utf8') as f: - code = f.read() - assert(isinstance(code, str)) - try: - codeblock = mparser.Parser(code, self.subdir).parse() - self.asts[subdir] = codeblock - except mesonlib.MesonException as me: - me.file = buildfilename - raise me - self.evaluate_codeblock(codeblock) - self.subdir = prev_subdir - - def func_files(self, node, args, kwargs): - if not isinstance(args, list): - return [args] - return args - - def transform(self): - self.load_root_meson_file() - self.asts[''] = self.ast - self.sanity_check_ast() - self.parse_project() - self.run() - - def add_source(self, targetname, filename): - self.operation = ADD_SOURCE - self.targetname = targetname - self.filename = filename - self.transform() - - def remove_source(self, targetname, filename): - self.operation = REMOVE_SOURCE - self.targetname = targetname - self.filename = filename - self.transform() - - def add_source_to_target(self, node, args, kwargs): - namespan = node.args.arguments[0].bytespan - buildfilename = os.path.join(self.source_root, self.subdir, environment.build_filename) - raw_data = open(buildfilename, 'r').read() - updated = raw_data[0:namespan[1]] + (", '%s'" % self.filename) + raw_data[namespan[1]:] - open(buildfilename, 'w').write(updated) - sys.exit(0) - - def remove_argument_item(self, args, i): - assert(isinstance(args, mparser.ArgumentNode)) - namespan = args.arguments[i].bytespan - # Usually remove the comma after this item but if it is - # the last argument, we need to remove the one before. - if i >= len(args.commas): - i -= 1 - if i < 0: - commaspan = (0, 0) # Removed every entry in the list. - else: - commaspan = args.commas[i].bytespan - if commaspan[0] < namespan[0]: - commaspan, namespan = namespan, commaspan - buildfilename = os.path.join(self.source_root, args.subdir, environment.build_filename) - raw_data = open(buildfilename, 'r').read() - intermediary = raw_data[0:commaspan[0]] + raw_data[commaspan[1]:] - updated = intermediary[0:namespan[0]] + intermediary[namespan[1]:] - open(buildfilename, 'w').write(updated) - sys.exit(0) - - def hacky_find_and_remove(self, node_to_remove): - for a in self.asts[node_to_remove.subdir].lines: - if a.lineno == node_to_remove.lineno: - if isinstance(a, mparser.AssignmentNode): - v = a.value - if not isinstance(v, mparser.ArrayNode): - raise NotImplementedError('Not supported yet, bro.') - args = v.args - for i in range(len(args.arguments)): - if isinstance(args.arguments[i], mparser.StringNode) and self.filename == args.arguments[i].value: - self.remove_argument_item(args, i) - raise NotImplementedError('Sukkess') - - def remove_source_from_target(self, node, args, kwargs): - for i in range(1, len(node.args)): - # Is file name directly in function call as a string. - if isinstance(node.args.arguments[i], mparser.StringNode) and self.filename == node.args.arguments[i].value: - self.remove_argument_item(node.args, i) - # Is file name in a variable that gets expanded here. - if isinstance(node.args.arguments[i], mparser.IdNode): - avar = self.get_variable(node.args.arguments[i].value) - if not isinstance(avar, list): - raise NotImplementedError('Non-arrays not supported yet, sorry.') - for entry in avar: - if isinstance(entry, mparser.StringNode) and entry.value == self.filename: - self.hacky_find_and_remove(entry) - sys.exit('Could not find source %s in target %s.' % (self.filename, args[0])) -- cgit v1.2.3 From 08da3873ddf81a97cdce782d8cde4b904280d8f5 Mon Sep 17 00:00:00 2001 From: Daniel Mensinger Date: Sun, 20 Jan 2019 19:27:42 +0100 Subject: Added target AST Interpreter support --- mesonbuild/ast/interpreter.py | 29 ++++++++++--- mesonbuild/ast/introspection.py | 94 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 115 insertions(+), 8 deletions(-) diff --git a/mesonbuild/ast/interpreter.py b/mesonbuild/ast/interpreter.py index 032337833..28f115039 100644 --- a/mesonbuild/ast/interpreter.py +++ b/mesonbuild/ast/interpreter.py @@ -170,16 +170,33 @@ class AstInterpreter(interpreterbase.InterpreterBase): def assignment(self, node): pass - def flatten_args(self, args): + def flatten_args(self, args, include_unknown_args: bool = False): # Resolve mparser.ArrayNode if needed flattend_args = [] + temp_args = [] if isinstance(args, mparser.ArrayNode): - args = [x.value for x in args.args.arguments] + args = [x for x in args.args.arguments] + elif isinstance(args, mparser.ArgumentNode): + args = [x for x in args.arguments] for i in args: if isinstance(i, mparser.ArrayNode): - flattend_args += [x.value for x in i.args.arguments] - elif isinstance(i, str): - flattend_args += [i] + temp_args += [x for x in i.args.arguments] else: - pass + temp_args += [i] + for i in temp_args: + if isinstance(i, mparser.ElementaryNode): + flattend_args += [i.value] + elif isinstance(i, (str, bool, int, float)) or include_unknown_args: + flattend_args += [i] return flattend_args + + def flatten_kwargs(self, kwargs: object, include_unknown_args: bool = False): + flattend_kwargs = {} + for key, val in kwargs.items(): + if isinstance(val, mparser.ElementaryNode): + flattend_kwargs[key] = val.value + elif isinstance(val, (mparser.ArrayNode, mparser.ArgumentNode)): + flattend_kwargs[key] = self.flatten_args(val, include_unknown_args) + elif isinstance(val, (str, bool, int, float)) or include_unknown_args: + flattend_kwargs[key] = val + return flattend_kwargs diff --git a/mesonbuild/ast/introspection.py b/mesonbuild/ast/introspection.py index f21788c44..fde9cc176 100644 --- a/mesonbuild/ast/introspection.py +++ b/mesonbuild/ast/introspection.py @@ -19,8 +19,9 @@ from . import AstInterpreter from .. import compilers, environment, mesonlib, mparser, optinterpreter from .. import coredata as cdata from ..interpreterbase import InvalidArguments +from ..build import Executable, CustomTarget, Jar, RunTarget, SharedLibrary, SharedModule, StaticLibrary -import os +import sys, os class IntrospectionHelper: # mimic an argparse namespace @@ -48,10 +49,18 @@ class IntrospectionInterpreter(AstInterpreter): self.backend = backend self.default_options = {'backend': self.backend} self.project_data = {} + self.targets = [] self.funcs.update({ + 'add_languages': self.func_add_languages, + 'executable': self.func_executable, + 'jar': self.func_jar, + 'library': self.func_library, 'project': self.func_project, - 'add_languages': self.func_add_languages + 'shared_library': self.func_shared_lib, + 'shared_module': self.func_shared_module, + 'static_library': self.func_static_lib, + 'both_libraries': self.func_both_lib, }) def func_project(self, node, args, kwargs): @@ -115,6 +124,87 @@ class IntrospectionInterpreter(AstInterpreter): if lang not in self.coredata.compilers: self.environment.detect_compilers(lang, need_cross_compiler) + def build_target(self, node, args, kwargs, targetclass): + if not args: + return + args = self.flatten_args(args, True) + kwargs = self.flatten_kwargs(kwargs, True) + name = args[0] + sources = args[1:] + if 'sources' in kwargs: + sources += self.flatten_args(kwargs['sources']) + + # Filter out kwargs from other target types. For example 'soversion' + # passed to library() when default_library == 'static'. + kwargs = {k: v for k, v in kwargs.items() if k in targetclass.known_kwargs} + + is_cross = False + objects = [] + target = targetclass(name, self.subdir, self.subproject, is_cross, sources, objects, self.environment, kwargs) + + self.targets += [{ + 'name': target.get_basename(), + 'id': target.get_id(), + 'type': target.get_typename(), + 'defined_in': os.path.normpath(os.path.join(self.source_root, self.subdir, environment.build_filename)), + 'subdir': self.subdir, + 'build_by_default': target.build_by_default, + 'sources': sources, + 'kwargs': kwargs, + 'node': node, + }] + + return + + def build_library(self, node, args, kwargs): + default_library = self.coredata.get_builtin_option('default_library') + if default_library == 'shared': + return self.build_target(node, args, kwargs, SharedLibrary) + elif default_library == 'static': + return self.build_target(node, args, kwargs, StaticLibrary) + elif default_library == 'both': + return self.build_target(node, args, kwargs, SharedLibrary) + + def func_executable(self, node, args, kwargs): + return self.build_target(node, args, kwargs, Executable) + + def func_static_lib(self, node, args, kwargs): + return self.build_target(node, args, kwargs, StaticLibrary) + + def func_shared_lib(self, node, args, kwargs): + return self.build_target(node, args, kwargs, SharedLibrary) + + def func_both_lib(self, node, args, kwargs): + return self.build_target(node, args, kwargs, SharedLibrary) + + def func_shared_module(self, node, args, kwargs): + return self.build_target(node, args, kwargs, SharedModule) + + def func_library(self, node, args, kwargs): + return self.build_library(node, args, kwargs) + + def func_jar(self, node, args, kwargs): + return self.build_target(node, args, kwargs, Jar) + + def func_build_target(self, node, args, kwargs): + if 'target_type' not in kwargs: + return + target_type = kwargs.pop('target_type') + if isinstance(target_type, mparser.ElementaryNode): + target_type = target_type.value + if target_type == 'executable': + return self.build_target(node, args, kwargs, Executable) + elif target_type == 'shared_library': + return self.build_target(node, args, kwargs, SharedLibrary) + elif target_type == 'static_library': + return self.build_target(node, args, kwargs, StaticLibrary) + elif target_type == 'both_libraries': + return self.build_target(node, args, kwargs, SharedLibrary) + elif target_type == 'library': + return self.build_library(node, args, kwargs) + elif target_type == 'jar': + return self.build_target(node, args, kwargs, Jar) + def is_subproject(self): return self.subproject != '' -- cgit v1.2.3 From 50bc0960e44a262906db3bde94c1daab7355d641 Mon Sep 17 00:00:00 2001 From: Daniel Mensinger Date: Sun, 20 Jan 2019 19:29:12 +0100 Subject: Updated basic rewrite testcase --- test cases/rewrite/1 basic/meson.build | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test cases/rewrite/1 basic/meson.build b/test cases/rewrite/1 basic/meson.build index a0485d079..b91767892 100644 --- a/test cases/rewrite/1 basic/meson.build +++ b/test cases/rewrite/1 basic/meson.build @@ -2,4 +2,5 @@ project('rewritetest', 'c') sources = ['trivial.c', 'notthere.c'] -exe = executable('trivialprog', sources) +exe1 = executable('trivialprog1', sources) +exe2 = executable('trivialprog2', ['main.cpp', 'fileA.cpp']) -- cgit v1.2.3 From f6339d6361c89f7b79cbf91e8eb56089f3c1a592 Mon Sep 17 00:00:00 2001 From: Daniel Mensinger Date: Mon, 21 Jan 2019 14:27:00 +0100 Subject: Added support for assignments in the AST interpretor --- mesonbuild/ast/interpreter.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/mesonbuild/ast/interpreter.py b/mesonbuild/ast/interpreter.py index 28f115039..8893e9b74 100644 --- a/mesonbuild/ast/interpreter.py +++ b/mesonbuild/ast/interpreter.py @@ -47,6 +47,7 @@ class AstInterpreter(interpreterbase.InterpreterBase): def __init__(self, source_root, subdir): super().__init__(source_root, subdir) self.visited_subdirs = {} + self.assignments = {} self.funcs.update({'project': self.func_do_nothing, 'test': self.func_do_nothing, 'benchmark': self.func_do_nothing, @@ -133,7 +134,11 @@ class AstInterpreter(interpreterbase.InterpreterBase): return 0 def evaluate_plusassign(self, node): - return 0 + assert(isinstance(node, mparser.PlusAssignmentNode)) + if node.var_name not in self.assignments: + self.assignments[node.var_name] = [] + self.assignments[node.var_name] += [node.value] # Save a reference to the value node + self.evaluate_statement(node.value) # Evaluate the value just in case def evaluate_indexing(self, node): return 0 @@ -168,7 +173,9 @@ class AstInterpreter(interpreterbase.InterpreterBase): return 0 def assignment(self, node): - pass + assert(isinstance(node, mparser.AssignmentNode)) + self.assignments[node.var_name] = [node.value] # Save a reference to the value node + self.evaluate_statement(node.value) # Evaluate the value just in case def flatten_args(self, args, include_unknown_args: bool = False): # Resolve mparser.ArrayNode if needed -- cgit v1.2.3 From 86d5799bc4d945927e26fdcb6e239905e0aa8146 Mon Sep 17 00:00:00 2001 From: Daniel Mensinger Date: Mon, 21 Jan 2019 21:52:18 +0100 Subject: First rewriter test case --- mesonbuild/ast/__init__.py | 3 +- mesonbuild/ast/interpreter.py | 2 +- mesonbuild/ast/introspection.py | 41 +++++++-- mesonbuild/rewriter.py | 154 ++++++++++++++++++++++++++++----- run_unittests.py | 99 ++++++++++----------- test cases/rewrite/1 basic/addSrc.json | 65 ++++++++++++++ test cases/rewrite/1 basic/info.json | 47 ++++++++++ test cases/rewrite/1 basic/meson.build | 18 +++- 8 files changed, 339 insertions(+), 90 deletions(-) create mode 100644 test cases/rewrite/1 basic/addSrc.json create mode 100644 test cases/rewrite/1 basic/info.json diff --git a/mesonbuild/ast/__init__.py b/mesonbuild/ast/__init__.py index 0d1a4d69d..a9370dc6e 100644 --- a/mesonbuild/ast/__init__.py +++ b/mesonbuild/ast/__init__.py @@ -22,10 +22,11 @@ __all__ = [ 'AstVisitor', 'AstPrinter', 'IntrospectionInterpreter', + 'build_target_functions', ] from .interpreter import AstInterpreter -from .introspection import IntrospectionInterpreter +from .introspection import IntrospectionInterpreter, build_target_functions from .visitor import AstVisitor from .postprocess import AstIDGenerator, AstIndentationGenerator from .printer import AstPrinter diff --git a/mesonbuild/ast/interpreter.py b/mesonbuild/ast/interpreter.py index 8893e9b74..81f6d5838 100644 --- a/mesonbuild/ast/interpreter.py +++ b/mesonbuild/ast/interpreter.py @@ -191,7 +191,7 @@ class AstInterpreter(interpreterbase.InterpreterBase): else: temp_args += [i] for i in temp_args: - if isinstance(i, mparser.ElementaryNode): + if isinstance(i, mparser.ElementaryNode) and not isinstance(i, mparser.IdNode): flattend_args += [i.value] elif isinstance(i, (str, bool, int, float)) or include_unknown_args: flattend_args += [i] diff --git a/mesonbuild/ast/introspection.py b/mesonbuild/ast/introspection.py index fde9cc176..11496db19 100644 --- a/mesonbuild/ast/introspection.py +++ b/mesonbuild/ast/introspection.py @@ -20,9 +20,11 @@ from .. import compilers, environment, mesonlib, mparser, optinterpreter from .. import coredata as cdata from ..interpreterbase import InvalidArguments from ..build import Executable, CustomTarget, Jar, RunTarget, SharedLibrary, SharedModule, StaticLibrary - +from pprint import pprint import sys, os +build_target_functions = ['executable', 'jar', 'library', 'shared_library', 'shared_module', 'static_library', 'both_libraries'] + class IntrospectionHelper: # mimic an argparse namespace def __init__(self, cross_file): @@ -127,12 +129,36 @@ class IntrospectionInterpreter(AstInterpreter): def build_target(self, node, args, kwargs, targetclass): if not args: return - args = self.flatten_args(args, True) kwargs = self.flatten_kwargs(kwargs, True) - name = args[0] - sources = args[1:] + name = self.flatten_args(args)[0] + srcqueue = [node] if 'sources' in kwargs: - sources += self.flatten_args(kwargs['sources']) + srcqueue += kwargs['sources'] + + source_nodes = [] + while srcqueue: + curr = srcqueue.pop(0) + arg_node = None + if isinstance(curr, mparser.FunctionNode): + arg_node = curr.args + elif isinstance(curr, mparser.ArrayNode): + arg_node = curr.args + elif isinstance(curr, mparser.IdNode): + # Try to resolve the ID and append the node to the queue + id = curr.value + if id in self.assignments and self.assignments[id]: + node = self.assignments[id][0] + if isinstance(node, (mparser.ArrayNode, mparser.IdNode, mparser.FunctionNode)): + srcqueue += [node] + if arg_node is None: + continue + elemetary_nodes = list(filter(lambda x: isinstance(x, (str, mparser.StringNode)), arg_node.arguments)) + srcqueue += list(filter(lambda x: isinstance(x, (mparser.FunctionNode, mparser.ArrayNode, mparser.IdNode)), arg_node.arguments)) + # Pop the first element if the function is a build target function + if isinstance(curr, mparser.FunctionNode) and curr.func_name in build_target_functions: + elemetary_nodes.pop(0) + if elemetary_nodes: + source_nodes += [curr] # Filter out kwargs from other target types. For example 'soversion' # passed to library() when default_library == 'static'. @@ -140,7 +166,8 @@ class IntrospectionInterpreter(AstInterpreter): is_cross = False objects = [] - target = targetclass(name, self.subdir, self.subproject, is_cross, sources, objects, self.environment, kwargs) + empty_sources = [] # Passing the unresolved sources list causes errors + target = targetclass(name, self.subdir, self.subproject, is_cross, empty_sources, objects, self.environment, kwargs) self.targets += [{ 'name': target.get_basename(), @@ -149,7 +176,7 @@ class IntrospectionInterpreter(AstInterpreter): 'defined_in': os.path.normpath(os.path.join(self.source_root, self.subdir, environment.build_filename)), 'subdir': self.subdir, 'build_by_default': target.build_by_default, - 'sources': sources, + 'sources': source_nodes, 'kwargs': kwargs, 'node': node, }] diff --git a/mesonbuild/rewriter.py b/mesonbuild/rewriter.py index cc5d2abef..37099cec5 100644 --- a/mesonbuild/rewriter.py +++ b/mesonbuild/rewriter.py @@ -23,36 +23,146 @@ # - move targets # - reindent? -from .ast import AstInterpreter, AstVisitor, AstIDGenerator, AstIndentationGenerator, AstPrinter +from .ast import IntrospectionInterpreter, build_target_functions, AstVisitor, AstIDGenerator, AstIndentationGenerator, AstPrinter from mesonbuild.mesonlib import MesonException -from mesonbuild import mlog +from . import mlog, mparser, environment import traceback +from functools import wraps +from pprint import pprint +import json, os + +class RewriterException(MesonException): + pass def add_arguments(parser): parser.add_argument('--sourcedir', default='.', help='Path to source directory.') parser.add_argument('-p', '--print', action='store_true', default=False, dest='print', help='Print the parsed AST.') + parser.add_argument('command', type=str) + +class RequiredKeys: + def __init__(self, keys): + self.keys = keys + + def __call__(self, f): + @wraps(f) + def wrapped(*wrapped_args, **wrapped_kwargs): + assert(len(wrapped_args) >= 2) + cmd = wrapped_args[1] + for key, val in self.keys.items(): + typ = val[0] # The type of the value + default = val[1] # The default value -- None is required + choices = val[2] # Valid choices -- None is for everything + if key not in cmd: + if default is not None: + cmd[key] = default + else: + raise RewriterException('Key "{}" is missing in object for {}' + .format(key, f.__name__)) + if not isinstance(cmd[key], typ): + raise RewriterException('Invalid type of "{}". Required is {} but provided was {}' + .format(key, typ.__name__, type(cmd[key]).__name__)) + if choices is not None: + assert(isinstance(choices, list)) + if cmd[key] not in choices: + raise RewriterException('Invalid value of "{}": Possible values are {} but provided was "{}"' + .format(key, choices, cmd[key])) + return f(*wrapped_args, **wrapped_kwargs) + + return wrapped + +rewriter_keys = { + 'target': { + 'target': (str, None, None), + 'operation': (str, None, ['src_add', 'src_rm', 'test']), + 'sources': (list, [], None), + 'debug': (bool, False, None) + } +} + +class Rewriter: + def __init__(self, sourcedir: str, generator: str = 'ninja'): + self.sourcedir = sourcedir + self.interpreter = IntrospectionInterpreter(sourcedir, '', generator) + self.id_generator = AstIDGenerator() + self.functions = { + 'target': self.process_target, + } + + def analyze_meson(self): + mlog.log('Analyzing meson file:', mlog.bold(os.path.join(self.sourcedir, environment.build_filename))) + self.interpreter.analyze() + mlog.log(' -- Project:', mlog.bold(self.interpreter.project_data['descriptive_name'])) + mlog.log(' -- Version:', mlog.cyan(self.interpreter.project_data['version'])) + self.interpreter.ast.accept(AstIndentationGenerator()) + self.interpreter.ast.accept(self.id_generator) + + def find_target(self, target: str): + for i in self.interpreter.targets: + if target == i['name'] or target == i['id']: + return i + return None + + @RequiredKeys(rewriter_keys['target']) + def process_target(self, cmd): + mlog.log('Processing target', mlog.bold(cmd['target']), 'operation', mlog.cyan(cmd['operation'])) + target = self.find_target(cmd['target']) + if target is None: + mlog.error('Unknown target "{}" --> skipping'.format(cmd['target'])) + if cmd['debug']: + pprint(self.interpreter.targets) + return + if cmd['debug']: + pprint(target) + + if cmd['operation'] == 'src_add': + mlog.warning('TODO') + elif cmd['operation'] == 'src_rm': + mlog.warning('TODO') + elif cmd['operation'] == 'test': + src_list = [] + for i in target['sources']: + args = [] + if isinstance(i, mparser.FunctionNode): + args = list(i.args.arguments) + if i.func_name in build_target_functions: + args.pop(0) + elif isinstance(i, mparser.ArrayNode): + args = i.args.arguments + elif isinstance(i, mparser.ArgumentNode): + args = i.arguments + for j in args: + if isinstance(j, mparser.StringNode): + src_list += [j.value] + test_data = { + 'name': target['name'], + 'sources': src_list + } + mlog.log(' !! target {}={}'.format(target['id'], json.dumps(test_data))) + + def process(self, cmd): + if 'type' not in cmd: + raise RewriterException('Command has no key "type"') + if cmd['type'] not in self.functions: + raise RewriterException('Unknown command "{}". Supported commands are: {}' + .format(cmd['type'], list(self.functions.keys()))) + self.functions[cmd['type']](cmd) def run(options): - rewriter = AstInterpreter(options.sourcedir, '') - try: - rewriter.load_root_meson_file() - rewriter.sanity_check_ast() - rewriter.parse_project() - rewriter.run() - - indentor = AstIndentationGenerator() - idgen = AstIDGenerator() - printer = AstPrinter() - rewriter.ast.accept(indentor) - rewriter.ast.accept(idgen) - rewriter.ast.accept(printer) - print(printer.result) - except Exception as e: - if isinstance(e, MesonException): - mlog.exception(e) - else: - traceback.print_exc() - return 1 + rewriter = Rewriter(options.sourcedir) + rewriter.analyze_meson() + if os.path.exists(options.command): + with open(options.command, 'r') as fp: + commands = json.load(fp) + else: + commands = json.loads(options.command) + + if not isinstance(commands, list): + raise TypeError('Command is not a list') + + for i in commands: + if not isinstance(i, object): + raise TypeError('Command is not an object') + rewriter.process(i) return 0 diff --git a/run_unittests.py b/run_unittests.py index 8c2ad12f2..f0ab40f4d 100755 --- a/run_unittests.py +++ b/run_unittests.py @@ -32,6 +32,7 @@ from unittest import mock from configparser import ConfigParser from glob import glob from pathlib import (PurePath, Path) +from distutils.dir_util import copy_tree import mesonbuild.mlog import mesonbuild.compilers @@ -995,6 +996,7 @@ class BasePlatformTests(unittest.TestCase): self.mconf_command = self.meson_command + ['configure'] self.mintro_command = self.meson_command + ['introspect'] self.wrap_command = self.meson_command + ['wrap'] + self.rewrite_command = self.meson_command + ['rewrite'] # Backend-specific build commands self.build_command, self.clean_command, self.test_command, self.install_command, \ self.uninstall_command = get_backend_commands(self.backend) @@ -1003,6 +1005,7 @@ class BasePlatformTests(unittest.TestCase): self.vala_test_dir = os.path.join(src_root, 'test cases/vala') self.framework_test_dir = os.path.join(src_root, 'test cases/frameworks') self.unit_test_dir = os.path.join(src_root, 'test cases/unit') + self.rewrite_test_dir = os.path.join(src_root, 'test cases/rewrite') # Misc stuff self.orig_env = os.environ.copy() if self.backend is Backend.ninja: @@ -4914,67 +4917,53 @@ class PythonTests(BasePlatformTests): class RewriterTests(BasePlatformTests): - def setUp(self): super().setUp() - src_root = os.path.dirname(__file__) - self.testroot = os.path.realpath(tempfile.mkdtemp()) - self.rewrite_command = python_command + [os.path.join(src_root, 'mesonrewriter.py')] - self.tmpdir = os.path.realpath(tempfile.mkdtemp()) - self.workdir = os.path.join(self.tmpdir, 'foo') - self.test_dir = os.path.join(src_root, 'test cases/rewrite') - - def tearDown(self): - windows_proof_rmtree(self.tmpdir) + self.maxDiff = None - def read_contents(self, fname): - with open(os.path.join(self.workdir, fname)) as f: - return f.read() + def prime(self, dirname): + copy_tree(os.path.join(self.rewrite_test_dir, dirname), self.builddir) - def check_effectively_same(self, mainfile, truth): - mf = self.read_contents(mainfile) - t = self.read_contents(truth) - # Rewriting is not guaranteed to do a perfect job of - # maintaining whitespace. - self.assertEqual(mf.replace(' ', ''), t.replace(' ', '')) + def rewrite(self, directory, args): + if isinstance(args, str): + args = [args] + out = subprocess.check_output(self.rewrite_command + ['--sourcedir', directory] + args, + universal_newlines=True) + return out - def prime(self, dirname): - shutil.copytree(os.path.join(self.test_dir, dirname), self.workdir) + data_regex = re.compile(r'^\s*!!\s*(\w+)\s+([^=]+)=(.*)$') + def extract_test_data(self, out): + lines = out.split('\n') + result = {} + for i in lines: + match = RewriterTests.data_regex.match(i) + if match: + typ = match.group(1) + id = match.group(2) + data = json.loads(match.group(3)) + if typ not in result: + result[typ] = {} + result[typ][id] = data + return result - def test_basic(self): + def test_target_source_list(self): self.prime('1 basic') - subprocess.check_call(self.rewrite_command + ['remove', - '--target=trivialprog', - '--filename=notthere.c', - '--sourcedir', self.workdir], - universal_newlines=True) - self.check_effectively_same('meson.build', 'removed.txt') - subprocess.check_call(self.rewrite_command + ['add', - '--target=trivialprog', - '--filename=notthere.c', - '--sourcedir', self.workdir], - universal_newlines=True) - self.check_effectively_same('meson.build', 'added.txt') - subprocess.check_call(self.rewrite_command + ['remove', - '--target=trivialprog', - '--filename=notthere.c', - '--sourcedir', self.workdir], - universal_newlines=True) - self.check_effectively_same('meson.build', 'removed.txt') - - def test_subdir(self): - self.prime('2 subdirs') - top = self.read_contents('meson.build') - s2 = self.read_contents('sub2/meson.build') - subprocess.check_call(self.rewrite_command + ['remove', - '--target=something', - '--filename=second.c', - '--sourcedir', self.workdir], - universal_newlines=True) - self.check_effectively_same('sub1/meson.build', 'sub1/after.txt') - self.assertEqual(top, self.read_contents('meson.build')) - self.assertEqual(s2, self.read_contents('sub2/meson.build')) - + out = self.rewrite(self.builddir, os.path.join(self.builddir, 'info.json')) + out = self.extract_test_data(out) + expected = { + 'target': { + 'trivialprog1@exe': {'name': 'trivialprog1', 'sources': ['main.cpp', 'fileA.cpp']}, + 'trivialprog2@exe': {'name': 'trivialprog2', 'sources': ['fileB.cpp', 'fileC.cpp']}, + 'trivialprog3@exe': {'name': 'trivialprog3', 'sources': ['main.cpp', 'fileA.cpp']}, + 'trivialprog4@exe': {'name': 'trivialprog4', 'sources': ['main.cpp', 'fileA.cpp']}, + 'trivialprog5@exe': {'name': 'trivialprog5', 'sources': ['main.cpp', 'fileB.cpp', 'fileC.cpp']}, + 'trivialprog6@exe': {'name': 'trivialprog6', 'sources': ['main.cpp', 'fileA.cpp']}, + 'trivialprog7@exe': {'name': 'trivialprog7', 'sources': ['fileB.cpp', 'fileC.cpp', 'main.cpp', 'fileA.cpp']}, + 'trivialprog8@exe': {'name': 'trivialprog8', 'sources': ['main.cpp', 'fileA.cpp']}, + 'trivialprog9@exe': {'name': 'trivialprog9', 'sources': ['main.cpp', 'fileA.cpp']}, + } + } + self.assertDictEqual(out, expected) class NativeFileTests(BasePlatformTests): @@ -5267,7 +5256,7 @@ def should_run_cross_mingw_tests(): def main(): unset_envs() cases = ['InternalTests', 'DataTests', 'AllPlatformTests', 'FailureTests', - 'PythonTests', 'NativeFileTests'] + 'PythonTests', 'NativeFileTests', 'RewriterTests'] if not is_windows(): cases += ['LinuxlikeTests'] if should_run_cross_arm_tests(): diff --git a/test cases/rewrite/1 basic/addSrc.json b/test cases/rewrite/1 basic/addSrc.json new file mode 100644 index 000000000..b609b0883 --- /dev/null +++ b/test cases/rewrite/1 basic/addSrc.json @@ -0,0 +1,65 @@ +[ + { + "type": "target", + "target": "trivialprog1", + "operation": "src_add", + "sources": ["added1.cpp", "added2.cpp"], + "debug": true + }, + { + "type": "target", + "target": "trivialprog2", + "operation": "src_add", + "sources": ["added1.cpp"], + "debug": true + }, + { + "type": "target", + "target": "trivialprog3", + "operation": "src_add", + "sources": ["added1.cpp", "added2.cpp", "added3.cpp"], + "debug": true + }, + { + "type": "target", + "target": "trivialprog4", + "operation": "src_add", + "sources": ["added1.cpp"], + "debug": true + }, + { + "type": "target", + "target": "trivialprog5", + "operation": "src_add", + "sources": ["added1.cpp"], + "debug": true + }, + { + "type": "target", + "target": "trivialprog6", + "operation": "src_add", + "sources": ["added1.cpp"], + "debug": true + }, + { + "type": "target", + "target": "trivialprog7", + "operation": "src_add", + "sources": ["added1.cpp"], + "debug": true + }, + { + "type": "target", + "target": "trivialprog8", + "operation": "src_add", + "sources": ["added1.cpp"], + "debug": true + }, + { + "type": "target", + "target": "trivialprog9", + "operation": "src_add", + "sources": ["added1.cpp"], + "debug": true + } +] diff --git a/test cases/rewrite/1 basic/info.json b/test cases/rewrite/1 basic/info.json new file mode 100644 index 000000000..be2a87384 --- /dev/null +++ b/test cases/rewrite/1 basic/info.json @@ -0,0 +1,47 @@ +[ + { + "type": "target", + "target": "trivialprog1", + "operation": "test" + }, + { + "type": "target", + "target": "trivialprog2", + "operation": "test" + }, + { + "type": "target", + "target": "trivialprog3", + "operation": "test" + }, + { + "type": "target", + "target": "trivialprog4", + "operation": "test" + }, + { + "type": "target", + "target": "trivialprog5", + "operation": "test" + }, + { + "type": "target", + "target": "trivialprog6", + "operation": "test" + }, + { + "type": "target", + "target": "trivialprog7", + "operation": "test" + }, + { + "type": "target", + "target": "trivialprog8", + "operation": "test" + }, + { + "type": "target", + "target": "trivialprog9", + "operation": "test" + } +] diff --git a/test cases/rewrite/1 basic/meson.build b/test cases/rewrite/1 basic/meson.build index b91767892..3e092ee60 100644 --- a/test cases/rewrite/1 basic/meson.build +++ b/test cases/rewrite/1 basic/meson.build @@ -1,6 +1,16 @@ -project('rewritetest', 'c') +project('rewritetest', 'cpp') -sources = ['trivial.c', 'notthere.c'] +src1 = ['main.cpp', 'fileA.cpp'] +src2 = files(['fileB.cpp', 'fileC.cpp']) +src3 = src1 +src4 = [src3] -exe1 = executable('trivialprog1', sources) -exe2 = executable('trivialprog2', ['main.cpp', 'fileA.cpp']) +exe1 = executable('trivialprog1', src1) +exe2 = executable('trivialprog2', [src2]) +exe3 = executable('trivialprog3', ['main.cpp', 'fileA.cpp']) +exe4 = executable('trivialprog4', ['main.cpp', ['fileA.cpp']]) +exe5 = executable('trivialprog5', [src2, 'main.cpp']) +exe6 = executable('trivialprog6', 'main.cpp', 'fileA.cpp') +exe7 = executable('trivialprog7', 'fileB.cpp', src1, 'fileC.cpp') +exe8 = executable('trivialprog8', src3) +exe9 = executable('trivialprog9', src4) -- cgit v1.2.3 From 0ce663239371576fcf4ad751d548cf2424525053 Mon Sep 17 00:00:00 2001 From: Daniel Mensinger Date: Mon, 21 Jan 2019 22:52:27 +0100 Subject: Added suport for adding sources to a target --- mesonbuild/rewriter.py | 27 ++++++++++++- run_unittests.py | 19 +++++++++ test cases/rewrite/1 basic/addSrc.json | 70 +++++++++++++++++++++------------- 3 files changed, 89 insertions(+), 27 deletions(-) diff --git a/mesonbuild/rewriter.py b/mesonbuild/rewriter.py index 37099cec5..9650d89ac 100644 --- a/mesonbuild/rewriter.py +++ b/mesonbuild/rewriter.py @@ -86,6 +86,7 @@ class Rewriter: self.sourcedir = sourcedir self.interpreter = IntrospectionInterpreter(sourcedir, '', generator) self.id_generator = AstIDGenerator() + self.modefied_nodes = [] self.functions = { 'target': self.process_target, } @@ -117,7 +118,31 @@ class Rewriter: pprint(target) if cmd['operation'] == 'src_add': - mlog.warning('TODO') + node = None + if target['sources']: + node = target['sources'][0] + else: + node = target['node'] + assert(node is not None) + + # Generate the new String nodes + to_append = [] + for i in cmd['sources']: + mlog.log(' -- Adding source', mlog.green(i), 'at', + mlog.yellow('{}:{}'.format(os.path.join(node.subdir, environment.build_filename), node.lineno))) + token = mparser.Token('string', node.subdir, 0, 0, 0, None, i) + to_append += [mparser.StringNode(token)] + + # Append to the AST at the right place + if isinstance(node, mparser.FunctionNode): + node.args.arguments += to_append + elif isinstance(node, mparser.ArrayNode): + node.args.arguments += to_append + elif isinstance(node, mparser.ArgumentNode): + node.arguments += to_append + + # Mark the node as modified + self.modefied_nodes += [node] elif cmd['operation'] == 'src_rm': mlog.warning('TODO') elif cmd['operation'] == 'test': diff --git a/run_unittests.py b/run_unittests.py index f0ab40f4d..0c9441818 100755 --- a/run_unittests.py +++ b/run_unittests.py @@ -4965,6 +4965,25 @@ class RewriterTests(BasePlatformTests): } self.assertDictEqual(out, expected) + def test_target_add_sources(self): + self.prime('1 basic') + out = self.rewrite(self.builddir, os.path.join(self.builddir, 'addSrc.json')) + out = self.extract_test_data(out) + expected = { + 'target': { + 'trivialprog1@exe': {'name': 'trivialprog1', 'sources': ['main.cpp', 'fileA.cpp', 'a1.cpp', 'a2.cpp', 'a6.cpp']}, + 'trivialprog2@exe': {'name': 'trivialprog2', 'sources': ['fileB.cpp', 'fileC.cpp']}, + 'trivialprog3@exe': {'name': 'trivialprog3', 'sources': ['main.cpp', 'fileA.cpp', 'a5.cpp']}, + 'trivialprog4@exe': {'name': 'trivialprog4', 'sources': ['main.cpp', 'a5.cpp', 'fileA.cpp']}, + 'trivialprog5@exe': {'name': 'trivialprog5', 'sources': ['main.cpp', 'a3.cpp', 'fileB.cpp', 'fileC.cpp']}, + 'trivialprog6@exe': {'name': 'trivialprog6', 'sources': ['main.cpp', 'fileA.cpp', 'a4.cpp']}, + 'trivialprog7@exe': {'name': 'trivialprog7', 'sources': ['fileB.cpp', 'fileC.cpp', 'main.cpp', 'fileA.cpp', 'a1.cpp', 'a2.cpp', 'a6.cpp']}, + 'trivialprog8@exe': {'name': 'trivialprog8', 'sources': ['main.cpp', 'fileA.cpp', 'a1.cpp', 'a2.cpp', 'a6.cpp']}, + 'trivialprog9@exe': {'name': 'trivialprog9', 'sources': ['main.cpp', 'fileA.cpp', 'a1.cpp', 'a2.cpp', 'a6.cpp']}, + } + } + self.assertDictEqual(out, expected) + class NativeFileTests(BasePlatformTests): def setUp(self): diff --git a/test cases/rewrite/1 basic/addSrc.json b/test cases/rewrite/1 basic/addSrc.json index b609b0883..6d8c5996c 100644 --- a/test cases/rewrite/1 basic/addSrc.json +++ b/test cases/rewrite/1 basic/addSrc.json @@ -3,63 +3,81 @@ "type": "target", "target": "trivialprog1", "operation": "src_add", - "sources": ["added1.cpp", "added2.cpp"], - "debug": true - }, - { - "type": "target", - "target": "trivialprog2", - "operation": "src_add", - "sources": ["added1.cpp"], - "debug": true + "sources": ["a1.cpp", "a2.cpp"] }, { "type": "target", "target": "trivialprog3", "operation": "src_add", - "sources": ["added1.cpp", "added2.cpp", "added3.cpp"], - "debug": true + "sources": ["a5.cpp"] }, { "type": "target", "target": "trivialprog4", "operation": "src_add", - "sources": ["added1.cpp"], - "debug": true + "sources": ["a5.cpp"] }, { "type": "target", "target": "trivialprog5", "operation": "src_add", - "sources": ["added1.cpp"], - "debug": true + "sources": ["a3.cpp"] }, { "type": "target", "target": "trivialprog6", "operation": "src_add", - "sources": ["added1.cpp"], - "debug": true + "sources": ["a4.cpp"] }, { "type": "target", - "target": "trivialprog7", + "target": "trivialprog9", "operation": "src_add", - "sources": ["added1.cpp"], - "debug": true + "sources": ["a6.cpp"] + }, + { + "type": "target", + "target": "trivialprog1", + "operation": "test" + }, + { + "type": "target", + "target": "trivialprog2", + "operation": "test" + }, + { + "type": "target", + "target": "trivialprog3", + "operation": "test" + }, + { + "type": "target", + "target": "trivialprog4", + "operation": "test" + }, + { + "type": "target", + "target": "trivialprog5", + "operation": "test" + }, + { + "type": "target", + "target": "trivialprog6", + "operation": "test" + }, + { + "type": "target", + "target": "trivialprog7", + "operation": "test" }, { "type": "target", "target": "trivialprog8", - "operation": "src_add", - "sources": ["added1.cpp"], - "debug": true + "operation": "test" }, { "type": "target", "target": "trivialprog9", - "operation": "src_add", - "sources": ["added1.cpp"], - "debug": true + "operation": "test" } ] -- cgit v1.2.3 From 8dd9b44831ec6b8de1c633ad5d366f3dea2df2cd Mon Sep 17 00:00:00 2001 From: Daniel Mensinger Date: Tue, 22 Jan 2019 11:12:36 +0100 Subject: Added support for removing sources from a target --- mesonbuild/rewriter.py | 69 ++++++++++++++++++++++++----- run_unittests.py | 19 ++++++++ test cases/rewrite/1 basic/rmSrc.json | 83 +++++++++++++++++++++++++++++++++++ 3 files changed, 159 insertions(+), 12 deletions(-) create mode 100644 test cases/rewrite/1 basic/rmSrc.json diff --git a/mesonbuild/rewriter.py b/mesonbuild/rewriter.py index 9650d89ac..6a1176776 100644 --- a/mesonbuild/rewriter.py +++ b/mesonbuild/rewriter.py @@ -117,7 +117,23 @@ class Rewriter: if cmd['debug']: pprint(target) + # Utility function to get a list of the sources from a node + def arg_list_from_node(n): + args = [] + if isinstance(n, mparser.FunctionNode): + args = list(n.args.arguments) + if n.func_name in build_target_functions: + args.pop(0) + elif isinstance(n, mparser.ArrayNode): + args = n.args.arguments + elif isinstance(n, mparser.ArgumentNode): + args = n.arguments + return args + if cmd['operation'] == 'src_add': + ################################# + ### Add sources to the target ### + ################################# node = None if target['sources']: node = target['sources'][0] @@ -142,22 +158,51 @@ class Rewriter: node.arguments += to_append # Mark the node as modified - self.modefied_nodes += [node] + if node not in self.modefied_nodes: + self.modefied_nodes += [node] elif cmd['operation'] == 'src_rm': - mlog.warning('TODO') + ###################################### + ### Remove sources from the target ### + ###################################### + # Helper to find the exact string node and its parent + def find_node(src): + for i in target['sources']: + for j in arg_list_from_node(i): + if isinstance(j, mparser.StringNode): + if j.value == src: + return i, j + return None, None + + for i in cmd['sources']: + # Try to find the node with the source string + root, string_node = find_node(i) + if root is None: + mlog.warning(' -- Unable to find source', mlog.green(i), 'in the target') + continue + + # Remove the found string node from the argument list + arg_node = None + if isinstance(root, mparser.FunctionNode): + arg_node = root.args + if isinstance(root, mparser.ArrayNode): + arg_node = root.args + if isinstance(root, mparser.ArgumentNode): + arg_node = root + assert(arg_node is not None) + mlog.log(' -- Removing source', mlog.green(i), 'from', + mlog.yellow('{}:{}'.format(os.path.join(string_node.subdir, environment.build_filename), string_node.lineno))) + arg_node.arguments.remove(string_node) + + # Mark the node as modified + if root not in self.modefied_nodes: + self.modefied_nodes += [root] elif cmd['operation'] == 'test': + ###################################### + ### List all sources in the target ### + ###################################### src_list = [] for i in target['sources']: - args = [] - if isinstance(i, mparser.FunctionNode): - args = list(i.args.arguments) - if i.func_name in build_target_functions: - args.pop(0) - elif isinstance(i, mparser.ArrayNode): - args = i.args.arguments - elif isinstance(i, mparser.ArgumentNode): - args = i.arguments - for j in args: + for j in arg_list_from_node(i): if isinstance(j, mparser.StringNode): src_list += [j.value] test_data = { diff --git a/run_unittests.py b/run_unittests.py index 0c9441818..9087a9c9d 100755 --- a/run_unittests.py +++ b/run_unittests.py @@ -4984,6 +4984,25 @@ class RewriterTests(BasePlatformTests): } self.assertDictEqual(out, expected) + def test_target_remove_sources(self): + self.prime('1 basic') + out = self.rewrite(self.builddir, os.path.join(self.builddir, 'rmSrc.json')) + out = self.extract_test_data(out) + expected = { + 'target': { + 'trivialprog1@exe': {'name': 'trivialprog1', 'sources': ['main.cpp']}, + 'trivialprog2@exe': {'name': 'trivialprog2', 'sources': ['fileC.cpp']}, + 'trivialprog3@exe': {'name': 'trivialprog3', 'sources': ['main.cpp']}, + 'trivialprog4@exe': {'name': 'trivialprog4', 'sources': ['main.cpp']}, + 'trivialprog5@exe': {'name': 'trivialprog5', 'sources': ['main.cpp', 'fileC.cpp']}, + 'trivialprog6@exe': {'name': 'trivialprog6', 'sources': ['main.cpp']}, + 'trivialprog7@exe': {'name': 'trivialprog7', 'sources': ['fileC.cpp', 'main.cpp']}, + 'trivialprog8@exe': {'name': 'trivialprog8', 'sources': ['main.cpp']}, + 'trivialprog9@exe': {'name': 'trivialprog9', 'sources': ['main.cpp']}, + } + } + self.assertDictEqual(out, expected) + class NativeFileTests(BasePlatformTests): def setUp(self): diff --git a/test cases/rewrite/1 basic/rmSrc.json b/test cases/rewrite/1 basic/rmSrc.json new file mode 100644 index 000000000..a8559a570 --- /dev/null +++ b/test cases/rewrite/1 basic/rmSrc.json @@ -0,0 +1,83 @@ +[ + { + "type": "target", + "target": "trivialprog1", + "operation": "src_rm", + "sources": ["fileA.cpp"] + }, + { + "type": "target", + "target": "trivialprog3", + "operation": "src_rm", + "sources": ["fileA.cpp"] + }, + { + "type": "target", + "target": "trivialprog4", + "operation": "src_rm", + "sources": ["fileA.cpp"] + }, + { + "type": "target", + "target": "trivialprog5", + "operation": "src_rm", + "sources": ["fileB.cpp"] + }, + { + "type": "target", + "target": "trivialprog6", + "operation": "src_rm", + "sources": ["fileA.cpp"] + }, + { + "type": "target", + "target": "trivialprog7", + "operation": "src_rm", + "sources": ["fileB.cpp"] + }, + { + "type": "target", + "target": "trivialprog1", + "operation": "test" + }, + { + "type": "target", + "target": "trivialprog2", + "operation": "test" + }, + { + "type": "target", + "target": "trivialprog3", + "operation": "test" + }, + { + "type": "target", + "target": "trivialprog4", + "operation": "test" + }, + { + "type": "target", + "target": "trivialprog5", + "operation": "test" + }, + { + "type": "target", + "target": "trivialprog6", + "operation": "test" + }, + { + "type": "target", + "target": "trivialprog7", + "operation": "test" + }, + { + "type": "target", + "target": "trivialprog8", + "operation": "test" + }, + { + "type": "target", + "target": "trivialprog9", + "operation": "test" + } +] -- cgit v1.2.3 From e089eb7665ca9dfb28e740829652add666f4a2f0 Mon Sep 17 00:00:00 2001 From: Daniel Mensinger Date: Tue, 22 Jan 2019 14:54:22 +0100 Subject: Fixed line and column numbers for dict and array nodes --- mesonbuild/mparser.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/mesonbuild/mparser.py b/mesonbuild/mparser.py index 845a1a115..ec188372d 100644 --- a/mesonbuild/mparser.py +++ b/mesonbuild/mparser.py @@ -262,17 +262,17 @@ class BreakNode(ElementaryNode): pass class ArrayNode(BaseNode): - def __init__(self, args): + def __init__(self, args, lineno, colno): self.subdir = args.subdir - self.lineno = args.lineno - self.colno = args.colno + self.lineno = lineno + self.colno = colno self.args = args class DictNode(BaseNode): - def __init__(self, args): + def __init__(self, args, lineno, colno): self.subdir = args.subdir - self.lineno = args.lineno - self.colno = args.colno + self.lineno = lineno + self.colno = colno self.args = args class EmptyNode(BaseNode): @@ -638,11 +638,11 @@ class Parser: elif self.accept('lbracket'): args = self.args() self.block_expect('rbracket', block_start) - return ArrayNode(args) + return ArrayNode(args, block_start.lineno, block_start.colno) elif self.accept('lcurl'): key_values = self.key_values() self.block_expect('rcurl', block_start) - return DictNode(key_values) + return DictNode(key_values, block_start.lineno, block_start.colno) else: return self.e9() -- cgit v1.2.3 From b7c6f3ec72c831c2af20eb5320d5f51b52d79227 Mon Sep 17 00:00:00 2001 From: Daniel Mensinger Date: Tue, 22 Jan 2019 16:00:10 +0100 Subject: Can now rewrite files --- mesonbuild/ast/printer.py | 16 ++++++- mesonbuild/rewriter.py | 86 ++++++++++++++++++++++++++++++++++ run_unittests.py | 14 +++++- test cases/rewrite/1 basic/addSrc.json | 6 +++ test cases/rewrite/1 basic/meson.build | 2 + 5 files changed, 120 insertions(+), 4 deletions(-) diff --git a/mesonbuild/ast/printer.py b/mesonbuild/ast/printer.py index aab5a301f..1f5814676 100644 --- a/mesonbuild/ast/printer.py +++ b/mesonbuild/ast/printer.py @@ -17,6 +17,7 @@ from .. import mparser from . import AstVisitor +import re arithmic_map = { 'add': '+', @@ -33,11 +34,18 @@ class AstPrinter(AstVisitor): self.arg_newline_cutoff = arg_newline_cutoff self.ci = '' self.is_newline = True + self.last_level = 0 + + def post_process(self): + self.result = re.sub(r'\s+\n', '\n', self.result) def append(self, data: str, node: mparser.BaseNode): level = 0 if node and hasattr(node, 'level'): level = node.level + else: + level = self.last_level + self.last_level = level if self.is_newline: self.result += ' ' * (level * self.indent) self.result += data @@ -179,13 +187,17 @@ class AstPrinter(AstVisitor): self.newline() for i in node.arguments: i.accept(self) - self.append(',', node) + self.append(', ', node) if break_args: self.newline() for key, val in node.kwargs.items(): self.append(key, node) self.appendS(':', node) val.accept(self) - self.append(',', node) + self.append(', ', node) if break_args: self.newline() + if break_args: + self.result = re.sub(r', \n$', '\n', self.result) + else: + self.result = re.sub(r', $', '', self.result) diff --git a/mesonbuild/rewriter.py b/mesonbuild/rewriter.py index 6a1176776..dfed1c63e 100644 --- a/mesonbuild/rewriter.py +++ b/mesonbuild/rewriter.py @@ -219,6 +219,90 @@ class Rewriter: .format(cmd['type'], list(self.functions.keys()))) self.functions[cmd['type']](cmd) + def apply_changes(self): + assert(all(hasattr(x, 'lineno') and hasattr(x, 'colno') and hasattr(x, 'subdir') for x in self.modefied_nodes)) + assert(all(isinstance(x, (mparser.ArrayNode, mparser.FunctionNode)) for x in self.modefied_nodes)) + # Sort based on line and column in reversed order + work_nodes = list(sorted(self.modefied_nodes, key=lambda x: x.lineno * 1000 + x.colno, reverse=True)) + + # Generating the new replacement string + str_list = [] + for i in work_nodes: + printer = AstPrinter() + i.accept(printer) + printer.post_process() + data = { + 'file': os.path.join(i.subdir, environment.build_filename), + 'str': printer.result.strip(), + 'node': i + } + str_list += [data] + + # Load build files + files = {} + for i in str_list: + if i['file'] in files: + continue + fpath = os.path.realpath(os.path.join(self.sourcedir, i['file'])) + fdata = '' + with open(fpath, 'r') as fp: + fdata = fp.read() + + # Generate line offsets numbers + m_lines = fdata.splitlines(True) + offset = 0 + line_offsets = [] + for j in m_lines: + line_offsets += [offset] + offset += len(j) + + files[i['file']] = { + 'path': fpath, + 'raw': fdata, + 'offsets': line_offsets + } + + # Replace in source code + for i in str_list: + offsets = files[i['file']]['offsets'] + raw = files[i['file']]['raw'] + node = i['node'] + line = node.lineno - 1 + col = node.colno + start = offsets[line]+col + end = start + if isinstance(node, mparser.ArrayNode): + if raw[end] != '[': + mlog.warning('Internal error: expected "[" at {}:{} but got "{}"'.format(line, col, raw[end])) + continue + counter = 1 + while counter > 0: + end += 1 + if raw[end] == '[': + counter += 1 + elif raw[end] == ']': + counter -= 1 + end += 1 + elif isinstance(node, mparser.FunctionNode): + while raw[end] != '(': + end += 1 + end += 1 + counter = 1 + while counter > 0: + end += 1 + if raw[end] == '(': + counter += 1 + elif raw[end] == ')': + counter -= 1 + end += 1 + raw = files[i['file']]['raw'] = raw[:start] + i['str'] + raw[end:] + + # Write the files back + for key, val in files.items(): + mlog.log('Rewriting', mlog.yellow(key)) + with open(val['path'], 'w') as fp: + fp.write(val['raw']) + def run(options): rewriter = Rewriter(options.sourcedir) rewriter.analyze_meson() @@ -235,4 +319,6 @@ def run(options): if not isinstance(i, object): raise TypeError('Command is not an object') rewriter.process(i) + + rewriter.apply_changes() return 0 diff --git a/run_unittests.py b/run_unittests.py index 9087a9c9d..c8d7f7a04 100755 --- a/run_unittests.py +++ b/run_unittests.py @@ -4972,10 +4972,10 @@ class RewriterTests(BasePlatformTests): expected = { 'target': { 'trivialprog1@exe': {'name': 'trivialprog1', 'sources': ['main.cpp', 'fileA.cpp', 'a1.cpp', 'a2.cpp', 'a6.cpp']}, - 'trivialprog2@exe': {'name': 'trivialprog2', 'sources': ['fileB.cpp', 'fileC.cpp']}, + 'trivialprog2@exe': {'name': 'trivialprog2', 'sources': ['fileB.cpp', 'fileC.cpp', 'a7.cpp']}, 'trivialprog3@exe': {'name': 'trivialprog3', 'sources': ['main.cpp', 'fileA.cpp', 'a5.cpp']}, 'trivialprog4@exe': {'name': 'trivialprog4', 'sources': ['main.cpp', 'a5.cpp', 'fileA.cpp']}, - 'trivialprog5@exe': {'name': 'trivialprog5', 'sources': ['main.cpp', 'a3.cpp', 'fileB.cpp', 'fileC.cpp']}, + 'trivialprog5@exe': {'name': 'trivialprog5', 'sources': ['main.cpp', 'a3.cpp', 'fileB.cpp', 'fileC.cpp', 'a7.cpp']}, 'trivialprog6@exe': {'name': 'trivialprog6', 'sources': ['main.cpp', 'fileA.cpp', 'a4.cpp']}, 'trivialprog7@exe': {'name': 'trivialprog7', 'sources': ['fileB.cpp', 'fileC.cpp', 'main.cpp', 'fileA.cpp', 'a1.cpp', 'a2.cpp', 'a6.cpp']}, 'trivialprog8@exe': {'name': 'trivialprog8', 'sources': ['main.cpp', 'fileA.cpp', 'a1.cpp', 'a2.cpp', 'a6.cpp']}, @@ -4984,6 +4984,11 @@ class RewriterTests(BasePlatformTests): } self.assertDictEqual(out, expected) + # Check the written file + out = self.rewrite(self.builddir, os.path.join(self.builddir, 'info.json')) + out = self.extract_test_data(out) + self.assertDictEqual(out, expected) + def test_target_remove_sources(self): self.prime('1 basic') out = self.rewrite(self.builddir, os.path.join(self.builddir, 'rmSrc.json')) @@ -5003,6 +5008,11 @@ class RewriterTests(BasePlatformTests): } self.assertDictEqual(out, expected) + # Check the written file + out = self.rewrite(self.builddir, os.path.join(self.builddir, 'info.json')) + out = self.extract_test_data(out) + self.assertDictEqual(out, expected) + class NativeFileTests(BasePlatformTests): def setUp(self): diff --git a/test cases/rewrite/1 basic/addSrc.json b/test cases/rewrite/1 basic/addSrc.json index 6d8c5996c..1a504bff2 100644 --- a/test cases/rewrite/1 basic/addSrc.json +++ b/test cases/rewrite/1 basic/addSrc.json @@ -5,6 +5,12 @@ "operation": "src_add", "sources": ["a1.cpp", "a2.cpp"] }, + { + "type": "target", + "target": "trivialprog2", + "operation": "src_add", + "sources": ["a7.cpp"] + }, { "type": "target", "target": "trivialprog3", diff --git a/test cases/rewrite/1 basic/meson.build b/test cases/rewrite/1 basic/meson.build index 3e092ee60..1bed0e18d 100644 --- a/test cases/rewrite/1 basic/meson.build +++ b/test cases/rewrite/1 basic/meson.build @@ -5,6 +5,8 @@ src2 = files(['fileB.cpp', 'fileC.cpp']) src3 = src1 src4 = [src3] +# Magic comment + exe1 = executable('trivialprog1', src1) exe2 = executable('trivialprog2', [src2]) exe3 = executable('trivialprog3', ['main.cpp', 'fileA.cpp']) -- cgit v1.2.3 From 027c3c9ac7d7003f816465cc76278dda17f4dd86 Mon Sep 17 00:00:00 2001 From: Daniel Mensinger Date: Tue, 22 Jan 2019 17:04:43 +0100 Subject: Added subdir test --- run_unittests.py | 16 ++++++++++++++++ test cases/rewrite/2 subdirs/addSrc.json | 13 +++++++++++++ test cases/rewrite/2 subdirs/info.json | 7 +++++++ test cases/rewrite/2 subdirs/meson.build | 1 - test cases/rewrite/2 subdirs/sub2/meson.build | 1 - 5 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 test cases/rewrite/2 subdirs/addSrc.json create mode 100644 test cases/rewrite/2 subdirs/info.json diff --git a/run_unittests.py b/run_unittests.py index c8d7f7a04..711621bab 100755 --- a/run_unittests.py +++ b/run_unittests.py @@ -5013,6 +5013,22 @@ class RewriterTests(BasePlatformTests): out = self.extract_test_data(out) self.assertDictEqual(out, expected) + def test_target_subdir(self): + self.prime('2 subdirs') + out = self.rewrite(self.builddir, os.path.join(self.builddir, 'addSrc.json')) + out = self.extract_test_data(out) + expected = { + 'target': { + 'd3a7449@@something@exe': {'name': 'something', 'sources': ['first.c', 'second.c', 'third.c']} + } + } + self.assertDictEqual(out, expected) + + # Check the written file + out = self.rewrite(self.builddir, os.path.join(self.builddir, 'info.json')) + out = self.extract_test_data(out) + self.assertDictEqual(out, expected) + class NativeFileTests(BasePlatformTests): def setUp(self): diff --git a/test cases/rewrite/2 subdirs/addSrc.json b/test cases/rewrite/2 subdirs/addSrc.json new file mode 100644 index 000000000..017476c60 --- /dev/null +++ b/test cases/rewrite/2 subdirs/addSrc.json @@ -0,0 +1,13 @@ +[ + { + "type": "target", + "target": "something", + "operation": "src_add", + "sources": ["third.c"] + }, + { + "type": "target", + "target": "something", + "operation": "test" + } +] diff --git a/test cases/rewrite/2 subdirs/info.json b/test cases/rewrite/2 subdirs/info.json new file mode 100644 index 000000000..01733333f --- /dev/null +++ b/test cases/rewrite/2 subdirs/info.json @@ -0,0 +1,7 @@ +[ + { + "type": "target", + "target": "something", + "operation": "test" + } +] diff --git a/test cases/rewrite/2 subdirs/meson.build b/test cases/rewrite/2 subdirs/meson.build index 79b7ad738..c7f3fec89 100644 --- a/test cases/rewrite/2 subdirs/meson.build +++ b/test cases/rewrite/2 subdirs/meson.build @@ -2,4 +2,3 @@ project('subdir rewrite', 'c') subdir('sub1') subdir('sub2') - diff --git a/test cases/rewrite/2 subdirs/sub2/meson.build b/test cases/rewrite/2 subdirs/sub2/meson.build index 0d92e7f42..44b4075ea 100644 --- a/test cases/rewrite/2 subdirs/sub2/meson.build +++ b/test cases/rewrite/2 subdirs/sub2/meson.build @@ -1,2 +1 @@ executable('something', srcs) - -- cgit v1.2.3 From a8477955a71cc3e657887c9673b1dc1f645f788f Mon Sep 17 00:00:00 2001 From: Daniel Mensinger Date: Tue, 22 Jan 2019 17:05:30 +0100 Subject: Removed old test files --- test cases/rewrite/1 basic/added.txt | 5 ----- test cases/rewrite/1 basic/removed.txt | 5 ----- test cases/rewrite/2 subdirs/sub1/after.txt | 1 - 3 files changed, 11 deletions(-) delete mode 100644 test cases/rewrite/1 basic/added.txt delete mode 100644 test cases/rewrite/1 basic/removed.txt delete mode 100644 test cases/rewrite/2 subdirs/sub1/after.txt diff --git a/test cases/rewrite/1 basic/added.txt b/test cases/rewrite/1 basic/added.txt deleted file mode 100644 index 657dd42fb..000000000 --- a/test cases/rewrite/1 basic/added.txt +++ /dev/null @@ -1,5 +0,0 @@ -project('rewritetest', 'c') - -sources = ['trivial.c'] - -exe = executable('trivialprog', 'notthere.c', sources) diff --git a/test cases/rewrite/1 basic/removed.txt b/test cases/rewrite/1 basic/removed.txt deleted file mode 100644 index 55192149c..000000000 --- a/test cases/rewrite/1 basic/removed.txt +++ /dev/null @@ -1,5 +0,0 @@ -project('rewritetest', 'c') - -sources = ['trivial.c'] - -exe = executable('trivialprog', sources) diff --git a/test cases/rewrite/2 subdirs/sub1/after.txt b/test cases/rewrite/2 subdirs/sub1/after.txt deleted file mode 100644 index 53ceaffd3..000000000 --- a/test cases/rewrite/2 subdirs/sub1/after.txt +++ /dev/null @@ -1 +0,0 @@ -srcs = ['first.c'] -- cgit v1.2.3 From 6fe2c2b209814d7fe94f60d54c38ae75b1dc67af Mon Sep 17 00:00:00 2001 From: Daniel Mensinger Date: Tue, 22 Jan 2019 17:31:15 +0100 Subject: Fixed flake8 issues --- mesonbuild/ast/interpreter.py | 4 ++-- mesonbuild/ast/introspection.py | 5 ++--- mesonbuild/rewriter.py | 5 ++--- run_unittests.py | 3 ++- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/mesonbuild/ast/interpreter.py b/mesonbuild/ast/interpreter.py index 81f6d5838..20714328b 100644 --- a/mesonbuild/ast/interpreter.py +++ b/mesonbuild/ast/interpreter.py @@ -18,7 +18,7 @@ from .. import interpreterbase, mparser, mesonlib from .. import environment -from ..interpreterbase import InterpreterException, InvalidArguments, BreakRequest, ContinueRequest +from ..interpreterbase import InvalidArguments, BreakRequest, ContinueRequest import os, sys @@ -204,6 +204,6 @@ class AstInterpreter(interpreterbase.InterpreterBase): flattend_kwargs[key] = val.value elif isinstance(val, (mparser.ArrayNode, mparser.ArgumentNode)): flattend_kwargs[key] = self.flatten_args(val, include_unknown_args) - elif isinstance(val, (str, bool, int, float)) or include_unknown_args: + elif isinstance(val, (str, bool, int, float)) or include_unknown_args: flattend_kwargs[key] = val return flattend_kwargs diff --git a/mesonbuild/ast/introspection.py b/mesonbuild/ast/introspection.py index 11496db19..5d0ec5aca 100644 --- a/mesonbuild/ast/introspection.py +++ b/mesonbuild/ast/introspection.py @@ -19,9 +19,8 @@ from . import AstInterpreter from .. import compilers, environment, mesonlib, mparser, optinterpreter from .. import coredata as cdata from ..interpreterbase import InvalidArguments -from ..build import Executable, CustomTarget, Jar, RunTarget, SharedLibrary, SharedModule, StaticLibrary -from pprint import pprint -import sys, os +from ..build import Executable, Jar, SharedLibrary, SharedModule, StaticLibrary +import os build_target_functions = ['executable', 'jar', 'library', 'shared_library', 'shared_module', 'static_library', 'both_libraries'] diff --git a/mesonbuild/rewriter.py b/mesonbuild/rewriter.py index dfed1c63e..f66da33b3 100644 --- a/mesonbuild/rewriter.py +++ b/mesonbuild/rewriter.py @@ -23,10 +23,9 @@ # - move targets # - reindent? -from .ast import IntrospectionInterpreter, build_target_functions, AstVisitor, AstIDGenerator, AstIndentationGenerator, AstPrinter +from .ast import IntrospectionInterpreter, build_target_functions, AstIDGenerator, AstIndentationGenerator, AstPrinter from mesonbuild.mesonlib import MesonException from . import mlog, mparser, environment -import traceback from functools import wraps from pprint import pprint import json, os @@ -269,7 +268,7 @@ class Rewriter: node = i['node'] line = node.lineno - 1 col = node.colno - start = offsets[line]+col + start = offsets[line] + col end = start if isinstance(node, mparser.ArrayNode): if raw[end] != '[': diff --git a/run_unittests.py b/run_unittests.py index 711621bab..e52a15ec8 100755 --- a/run_unittests.py +++ b/run_unittests.py @@ -4917,6 +4917,8 @@ class PythonTests(BasePlatformTests): class RewriterTests(BasePlatformTests): + data_regex = re.compile(r'^\s*!!\s*(\w+)\s+([^=]+)=(.*)$') + def setUp(self): super().setUp() self.maxDiff = None @@ -4931,7 +4933,6 @@ class RewriterTests(BasePlatformTests): universal_newlines=True) return out - data_regex = re.compile(r'^\s*!!\s*(\w+)\s+([^=]+)=(.*)$') def extract_test_data(self, out): lines = out.split('\n') result = {} -- cgit v1.2.3 From 4b3e21b8941fe53b85eb889734c34937438b0928 Mon Sep 17 00:00:00 2001 From: Daniel Mensinger Date: Tue, 22 Jan 2019 17:47:51 +0100 Subject: Fixed test case and setup.py --- run_unittests.py | 10 +++------- setup.py | 1 + 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/run_unittests.py b/run_unittests.py index e52a15ec8..cde5589c7 100755 --- a/run_unittests.py +++ b/run_unittests.py @@ -5018,17 +5018,13 @@ class RewriterTests(BasePlatformTests): self.prime('2 subdirs') out = self.rewrite(self.builddir, os.path.join(self.builddir, 'addSrc.json')) out = self.extract_test_data(out) - expected = { - 'target': { - 'd3a7449@@something@exe': {'name': 'something', 'sources': ['first.c', 'second.c', 'third.c']} - } - } - self.assertDictEqual(out, expected) + expected = {'name': 'something', 'sources': ['first.c', 'second.c', 'third.c']} + self.assertDictEqual(list(out['target'].values())[0], expected) # Check the written file out = self.rewrite(self.builddir, os.path.join(self.builddir, 'info.json')) out = self.extract_test_data(out) - self.assertDictEqual(out, expected) + self.assertDictEqual(list(out['target'].values())[0], expected) class NativeFileTests(BasePlatformTests): diff --git a/setup.py b/setup.py index f1f2e81c1..f35296030 100644 --- a/setup.py +++ b/setup.py @@ -28,6 +28,7 @@ from setuptools import setup # Other platforms will create bin/meson entries = {'console_scripts': ['meson=mesonbuild.mesonmain:main']} packages = ['mesonbuild', + 'mesonbuild.ast', 'mesonbuild.backend', 'mesonbuild.compilers', 'mesonbuild.dependencies', -- cgit v1.2.3 From dbb94f122ddeb5a37ff2603acbcc701b996958bb Mon Sep 17 00:00:00 2001 From: Daniel Mensinger Date: Thu, 24 Jan 2019 21:38:29 +0100 Subject: Fixed style issues --- mesonbuild/ast/printer.py | 28 ++++++++++++++-------------- mesonbuild/rewriter.py | 12 +++--------- 2 files changed, 17 insertions(+), 23 deletions(-) diff --git a/mesonbuild/ast/printer.py b/mesonbuild/ast/printer.py index 1f5814676..60e0b0d94 100644 --- a/mesonbuild/ast/printer.py +++ b/mesonbuild/ast/printer.py @@ -51,7 +51,7 @@ class AstPrinter(AstVisitor): self.result += data self.is_newline = False - def appendS(self, data: str, node: mparser.BaseNode): + def append_padded(self, data: str, node: mparser.BaseNode): if self.result[-1] not in [' ', '\n']: data = ' ' + data self.append(data + ' ', node) @@ -90,26 +90,26 @@ class AstPrinter(AstVisitor): def visit_OrNode(self, node: mparser.OrNode): node.left.accept(self) - self.appendS('or', node) + self.append_padded('or', node) node.right.accept(self) def visit_AndNode(self, node: mparser.AndNode): node.left.accept(self) - self.appendS('and', node) + self.append_padded('and', node) node.right.accept(self) def visit_ComparisonNode(self, node: mparser.ComparisonNode): node.left.accept(self) - self.appendS(mparser.comparison_map[node.ctype], node) + self.append_padded(mparser.comparison_map[node.ctype], node) node.right.accept(self) def visit_ArithmeticNode(self, node: mparser.ArithmeticNode): node.left.accept(self) - self.appendS(arithmic_map[node.operation], node) + self.append_padded(arithmic_map[node.operation], node) node.right.accept(self) def visit_NotNode(self, node: mparser.NotNode): - self.appendS('not', node) + self.append_padded('not', node) node.value.accept(self) def visit_CodeBlockNode(self, node: mparser.CodeBlockNode): @@ -143,9 +143,9 @@ class AstPrinter(AstVisitor): def visit_ForeachClauseNode(self, node: mparser.ForeachClauseNode): varnames = [x.value for x in node.varnames] - self.appendS('foreach', node) - self.appendS(', '.join(varnames), node) - self.appendS(':', node) + self.append_padded('foreach', node) + self.append_padded(', '.join(varnames), node) + self.append_padded(':', node) node.items.accept(self) self.newline() node.block.accept(self) @@ -154,7 +154,7 @@ class AstPrinter(AstVisitor): def visit_IfClauseNode(self, node: mparser.IfClauseNode): prefix = '' for i in node.ifs: - self.appendS(prefix + 'if', node) + self.append_padded(prefix + 'if', node) prefix = 'el' i.accept(self) if node.elseblock: @@ -163,7 +163,7 @@ class AstPrinter(AstVisitor): self.append('endif', node) def visit_UMinusNode(self, node: mparser.UMinusNode): - self.appendS('-', node) + self.append_padded('-', node) node.value.accept(self) def visit_IfNode(self, node: mparser.IfNode): @@ -173,9 +173,9 @@ class AstPrinter(AstVisitor): def visit_TernaryNode(self, node: mparser.TernaryNode): node.condition.accept(self) - self.appendS('?', node) + self.append_padded('?', node) node.trueblock.accept(self) - self.appendS(':', node) + self.append_padded(':', node) node.falseblock.accept(self) def visit_ArgumentNode(self, node: mparser.ArgumentNode): @@ -192,7 +192,7 @@ class AstPrinter(AstVisitor): self.newline() for key, val in node.kwargs.items(): self.append(key, node) - self.appendS(':', node) + self.append_padded(':', node) val.accept(self) self.append(', ', node) if break_args: diff --git a/mesonbuild/rewriter.py b/mesonbuild/rewriter.py index f66da33b3..277835c7c 100644 --- a/mesonbuild/rewriter.py +++ b/mesonbuild/rewriter.py @@ -130,9 +130,6 @@ class Rewriter: return args if cmd['operation'] == 'src_add': - ################################# - ### Add sources to the target ### - ################################# node = None if target['sources']: node = target['sources'][0] @@ -159,10 +156,8 @@ class Rewriter: # Mark the node as modified if node not in self.modefied_nodes: self.modefied_nodes += [node] + elif cmd['operation'] == 'src_rm': - ###################################### - ### Remove sources from the target ### - ###################################### # Helper to find the exact string node and its parent def find_node(src): for i in target['sources']: @@ -195,10 +190,9 @@ class Rewriter: # Mark the node as modified if root not in self.modefied_nodes: self.modefied_nodes += [root] + elif cmd['operation'] == 'test': - ###################################### - ### List all sources in the target ### - ###################################### + # List all sources in the target src_list = [] for i in target['sources']: for j in arg_list_from_node(i): -- cgit v1.2.3