diff options
| -rw-r--r-- | mesonbuild/ast/interpreter.py | 20 | ||||
| -rw-r--r-- | mesonbuild/ast/introspection.py | 50 | ||||
| -rw-r--r-- | mesonbuild/mintro.py | 44 | ||||
| -rw-r--r-- | mesonbuild/mparser.py | 7 | ||||
| -rw-r--r-- | mesonbuild/rewriter.py | 366 | ||||
| -rw-r--r-- | test cases/rewrite/1 basic/addSrc.json | 38 | ||||
| -rw-r--r-- | test cases/rewrite/1 basic/addTgt.json | 2 | ||||
| -rw-r--r-- | test cases/rewrite/1 basic/expected_dag.txt | 160 | ||||
| -rw-r--r-- | test cases/rewrite/1 basic/info.json | 20 | ||||
| -rw-r--r-- | test cases/rewrite/1 basic/meson.build | 16 | ||||
| -rw-r--r-- | test cases/rewrite/1 basic/rmSrc.json | 40 | ||||
| -rw-r--r-- | test cases/rewrite/1 basic/rmTgt.json | 5 | ||||
| -rw-r--r-- | test cases/rewrite/7 tricky dataflow/addSrc.json | 72 | ||||
| -rw-r--r-- | test cases/rewrite/7 tricky dataflow/info.json | 32 | ||||
| -rw-r--r-- | test cases/rewrite/7 tricky dataflow/meson.build | 35 | ||||
| -rw-r--r-- | unittests/allplatformstests.py | 4 | ||||
| -rw-r--r-- | unittests/rewritetests.py | 115 |
17 files changed, 715 insertions, 311 deletions
diff --git a/mesonbuild/ast/interpreter.py b/mesonbuild/ast/interpreter.py index e38504d4e..473ad9119 100644 --- a/mesonbuild/ast/interpreter.py +++ b/mesonbuild/ast/interpreter.py @@ -11,6 +11,7 @@ import typing as T from collections import defaultdict from dataclasses import dataclass import itertools +from pathlib import Path from .. import mparser, mesonlib from .. import environment @@ -28,6 +29,8 @@ from ..interpreterbase import ( InterpreterObject, ) +from ..interpreterbase.helpers import flatten + from ..interpreter import ( StringHolder, BooleanHolder, @@ -54,7 +57,6 @@ from ..mparser import ( ) if T.TYPE_CHECKING: - from pathlib import Path from .visitor import AstVisitor from ..interpreter import Interpreter from ..interpreterbase import SubProject, TYPE_var, TYPE_nvar @@ -110,7 +112,7 @@ class IntrospectionBuildTarget(MesonInterpreterObject): installed: bool outputs: T.List[str] source_nodes: T.List[BaseNode] - extra_files: T.List[BaseNode] + extra_files: BaseNode kwargs: T.Dict[str, TYPE_var] node: FunctionNode @@ -695,6 +697,20 @@ class AstInterpreter(InterpreterBase): assert isinstance(var_name, str) self.cur_assignments[var_name].append((self.nesting.copy(), node)) + def nodes_to_pretty_filelist(self, root_path: Path, subdir: str, nodes: T.List[BaseNode]) -> T.List[T.Union[str, UnknownValue]]: + def src_to_abs(src: T.Union[str, IntrospectionFile, UnknownValue]) -> T.Union[str, UnknownValue]: + if isinstance(src, str): + return os.path.normpath(os.path.join(root_path, subdir, src)) + elif isinstance(src, IntrospectionFile): + return str(src.to_abs_path(root_path)) + elif isinstance(src, UnknownValue): + return src + else: + raise TypeError + + rtvals: T.List[T.Any] = flatten([self.node_to_runtime_value(sn) for sn in nodes]) + return [src_to_abs(x) for x in rtvals] + def flatten_args(self, args_raw: T.Union[TYPE_nvar, T.Sequence[TYPE_nvar]], include_unknown_args: bool = False) -> T.List[TYPE_var]: # Make sure we are always dealing with lists if isinstance(args_raw, list): diff --git a/mesonbuild/ast/introspection.py b/mesonbuild/ast/introspection.py index 26caa727e..147436d4f 100644 --- a/mesonbuild/ast/introspection.py +++ b/mesonbuild/ast/introspection.py @@ -15,7 +15,7 @@ from ..compilers import detect_compiler_for from ..interpreterbase import InvalidArguments, SubProject, UnknownValue from ..mesonlib import MachineChoice from ..options import OptionKey -from ..mparser import BaseNode, ArithmeticNode, ArrayNode, ElementaryNode, IdNode, FunctionNode, StringNode +from ..mparser import BaseNode, ArrayNode, ElementaryNode, IdNode, FunctionNode, StringNode from .interpreter import AstInterpreter, IntrospectionBuildTarget, IntrospectionDependency if T.TYPE_CHECKING: @@ -237,40 +237,20 @@ class IntrospectionInterpreter(AstInterpreter): kwargs = self.flatten_kwargs(kwargs_raw, True) - def traverse_nodes(inqueue: T.List[BaseNode]) -> T.List[BaseNode]: - res: T.List[BaseNode] = [] - while inqueue: - curr = inqueue.pop(0) - arg_node = None - assert isinstance(curr, BaseNode) - if isinstance(curr, FunctionNode): - arg_node = curr.args - elif isinstance(curr, ArrayNode): - arg_node = curr.args - elif isinstance(curr, IdNode): - # Try to resolve the ID and append the node to the queue - assert isinstance(curr.value, str) - var_name = curr.value - if var_name in self.cur_assignments: - tmp_node = self.get_cur_value(var_name) - if isinstance(tmp_node, (ArrayNode, IdNode, FunctionNode)): - inqueue += [tmp_node] - elif isinstance(curr, ArithmeticNode): - inqueue += [curr.left, curr.right] - if arg_node is None: - continue - arg_nodes = arg_node.arguments.copy() - # Pop the first element if the function is a build target function - if isinstance(curr, FunctionNode) and curr.func_name.value in BUILD_TARGET_FUNCTIONS: - arg_nodes.pop(0) - elementary_nodes = [x for x in arg_nodes if isinstance(x, (str, StringNode))] - inqueue += [x for x in arg_nodes if isinstance(x, (FunctionNode, ArrayNode, IdNode, ArithmeticNode))] - if elementary_nodes: - res += [curr] - return res - - source_nodes = traverse_nodes(srcqueue) - extraf_nodes = traverse_nodes(extra_queue) + oldlen = len(node.args.arguments) + source_nodes = node.args.arguments[1:] + for k, v in node.args.kwargs.items(): + assert isinstance(k, IdNode) + if k.value == 'sources': + source_nodes.append(v) + assert oldlen == len(node.args.arguments) + + extraf_nodes = None + for k, v in node.args.kwargs.items(): + assert isinstance(k, IdNode) + if k.value == 'extra_files': + assert extraf_nodes is None + extraf_nodes = v # Make sure nothing can crash when creating the build class kwargs_reduced = {k: v for k, v in kwargs.items() if k in targetclass.known_kwargs and k in {'install', 'build_by_default', 'build_always', 'name_prefix'}} diff --git a/mesonbuild/mintro.py b/mesonbuild/mintro.py index 09d373652..db62dc06c 100644 --- a/mesonbuild/mintro.py +++ b/mesonbuild/mintro.py @@ -19,20 +19,17 @@ from pathlib import Path, PurePath import sys import typing as T -from . import build, mesonlib, options, coredata as cdata -from .ast import IntrospectionInterpreter, BUILD_TARGET_FUNCTIONS, AstConditionLevel, AstIDGenerator, AstIndentationGenerator, AstJSONPrinter +from . import build, environment, mesonlib, options, coredata as cdata +from .ast import IntrospectionInterpreter, AstConditionLevel, AstIDGenerator, AstIndentationGenerator, AstJSONPrinter from .backend import backends from .dependencies import Dependency -from . import environment from .interpreterbase import ObjectHolder, UnknownValue from .options import OptionKey -from .mparser import FunctionNode, ArrayNode, ArgumentNode, StringNode if T.TYPE_CHECKING: import argparse from .interpreter import Interpreter - from .mparser import BaseNode def get_meson_info_file(info_dir: str) -> str: return os.path.join(info_dir, 'meson-info.json') @@ -169,35 +166,14 @@ def get_target_dir(coredata: cdata.CoreData, subdir: str) -> str: else: return subdir -def list_targets_from_source(intr: IntrospectionInterpreter) -> T.List[T.Dict[str, T.Union[bool, str, T.List[T.Union[str, T.Dict[str, T.Union[str, T.List[str], bool]]]]]]]: - tlist: T.List[T.Dict[str, T.Union[bool, str, T.List[T.Union[str, T.Dict[str, T.Union[str, T.List[str], bool]]]]]]] = [] - root_dir = Path(intr.source_root) - - def nodes_to_paths(node_list: T.List[BaseNode]) -> T.List[Path]: - res: T.List[Path] = [] - for n in node_list: - args: T.List[BaseNode] = [] - if isinstance(n, FunctionNode): - args = list(n.args.arguments) - if n.func_name.value in BUILD_TARGET_FUNCTIONS: - args.pop(0) - elif isinstance(n, ArrayNode): - args = n.args.arguments - elif isinstance(n, ArgumentNode): - args = n.arguments - for j in args: - if isinstance(j, StringNode): - assert isinstance(j.value, str) - res += [Path(j.value)] - elif isinstance(j, str): - res += [Path(j)] - res = [root_dir / i.subdir / x for x in res] - res = [x.resolve() for x in res] - return res +def list_targets_from_source(intr: IntrospectionInterpreter) -> T.List[T.Dict[str, object]]: + tlist = [] + root_dir = Path(intr.source_root).resolve() for i in intr.targets: - sources = nodes_to_paths(i.source_nodes) - extra_f = nodes_to_paths(i.extra_files) + sources = intr.nodes_to_pretty_filelist(root_dir, i.subdir, i.source_nodes) + extra_files = intr.nodes_to_pretty_filelist(root_dir, i.subdir, [i.extra_files] if i.extra_files else []) + outdir = get_target_dir(intr.coredata, i.subdir) tlist += [{ @@ -212,11 +188,11 @@ def list_targets_from_source(intr: IntrospectionInterpreter) -> T.List[T.Dict[st 'machine': i.machine, 'compiler': [], 'parameters': [], - 'sources': [str(x) for x in sources], + 'sources': sources, 'generated_sources': [] }], 'depends': [], - 'extra_files': [str(x) for x in extra_f], + 'extra_files': extra_files, 'subproject': None, # Subprojects are not supported 'installed': i.installed }] diff --git a/mesonbuild/mparser.py b/mesonbuild/mparser.py index a0a8d273e..3dd8f0a5e 100644 --- a/mesonbuild/mparser.py +++ b/mesonbuild/mparser.py @@ -369,6 +369,13 @@ class ArgumentNode(BaseNode): mlog.warning('This will be an error in Meson 2.0.') self.kwargs[name] = value + def get_kwarg_or_default(self, name: str, default: BaseNode) -> BaseNode: + for k, v in self.kwargs.items(): + assert isinstance(k, IdNode) + if k.value == name: + return v + return default + def set_kwarg_no_check(self, name: BaseNode, value: BaseNode) -> None: self.kwargs[name] = value diff --git a/mesonbuild/rewriter.py b/mesonbuild/rewriter.py index 981dfd326..9023033b9 100644 --- a/mesonbuild/rewriter.py +++ b/mesonbuild/rewriter.py @@ -11,13 +11,15 @@ from __future__ import annotations from .ast import IntrospectionInterpreter, BUILD_TARGET_FUNCTIONS, AstConditionLevel, AstIDGenerator, AstIndentationGenerator, AstPrinter from .ast.interpreter import IntrospectionBuildTarget, IntrospectionDependency, _symbol -from .interpreterbase import TV_func -from mesonbuild.mesonlib import MesonException, setup_vsenv +from .interpreterbase import UnknownValue, TV_func +from .interpreterbase.helpers import flatten +from mesonbuild.mesonlib import MesonException, setup_vsenv, relpath from . import mlog, environment from functools import wraps -from .mparser import Token, ArrayNode, ArgumentNode, AssignmentNode, BaseNode, StringNode, BooleanNode, ElementaryNode, IdNode, FunctionNode, PlusAssignmentNode -import json, os, re, sys +from .mparser import Token, ArrayNode, ArgumentNode, ArithmeticNode, AssignmentNode, BaseNode, StringNode, BooleanNode, ElementaryNode, IdNode, FunctionNode, PlusAssignmentNode +import json, os, re, sys, codecs import typing as T +from pathlib import Path if T.TYPE_CHECKING: import argparse @@ -633,6 +635,166 @@ class Rewriter: return ass return None + def affects_no_other_targets(self, candidate: BaseNode) -> bool: + affected = self.interpreter.dataflow_dag.reachable({candidate}, False) + affected_targets = [x for x in affected if isinstance(x, FunctionNode) and x.func_name.value in BUILD_TARGET_FUNCTIONS] + return len(affected_targets) == 1 + + def get_relto(self, target_node: BaseNode, node: BaseNode) -> Path: + cwd = Path(os.getcwd()) + all_paths = self.interpreter.dataflow_dag.find_all_paths(node, target_node) + # len(all_paths) == 0 would imply that data does not flow from node to + # target_node. This would imply that adding sources to node would not + # add the source to the target. + assert all_paths + if len(all_paths) > 1: + return None + return (cwd / next(x for x in all_paths[0] if isinstance(x, FunctionNode)).filename).parent + + def add_src_or_extra(self, op: str, target: IntrospectionBuildTarget, newfiles: T.List[str], to_sort_nodes: T.List[T.Union[FunctionNode, ArrayNode]]) -> None: + assert op in {'src_add', 'extra_files_add'} + + if op == 'src_add': + old: T.Set[T.Union[BaseNode, UnknownValue]] = set(target.source_nodes) + elif op == 'extra_files_add': + if target.extra_files is None: + old = set() + else: + old = {target.extra_files} + tgt_function: FunctionNode = target.node + + cwd = Path(os.getcwd()) + target_dir_abs = cwd / os.path.dirname(target.node.filename) + source_root_abs = cwd / self.interpreter.source_root + + candidates1 = self.interpreter.dataflow_dag.reachable(old, True) + # A node is a member of the set `candidates1` exactly if data from this node + # flow into one of the `dest` nodes. We assume that this implies that if we + # add `foo.c` to this node, then 'foo.c' will be added to one of these + # nodes. This assumption is not always true: + # ar = ['a.c', 'b.c'] + # srcs = ar[1] + # executable('name', srcs) + # Data flows from `ar` to `srcs`, but if we add 'foo.c': + # ar = ['a.c', 'b.c', 'foo.c'] + # srcs = ar[1] + # executable('name', srcs) + # this does not add 'foo.c' to `srcs`. This is a known bug/limitation of + # the meson rewriter that could be fixed by replacing `reachable` with a + # more advanced analysis. But this is a lot of work and I think e.g. + # `srcs = ar[1]` is rare in real-world projects, so I will just leave + # this for now. + + candidates2 = {x for x in candidates1 if isinstance(x, (FunctionNode, ArrayNode))} + + # If we have this meson.build file: + # shared = ['shared.c'] + # executable('foo', shared + ['foo.c']) + # executable('bar', shared + ['bar.c']) + # and we are tasked with adding 'new.c' to 'foo', we should do e.g this: + # shared = ['shared.c'] + # executable('foo', shared + ['foo.c', 'new.c']) + # executable('bar', shared + ['bar.c']) + # but never this: + # shared = ['shared.c', 'new.c'] + # executable('foo', shared + ['foo.c']) + # executable('bar', shared + ['bar.c']) + # We do this by removing the `['shared.c']`-node from `candidates2`. + candidates2 = {x for x in candidates2 if self.affects_no_other_targets(x)} + + def path_contains_unknowns(candidate: BaseNode) -> bool: + all_paths = self.interpreter.dataflow_dag.find_all_paths(candidate, target.node) + for path in all_paths: + for el in path: + if isinstance(el, UnknownValue): + return True + return False + + candidates2 = {x for x in candidates2 if not path_contains_unknowns(x)} + + candidates2 = {x for x in candidates2 if self.get_relto(target.node, x) is not None} + + chosen: T.Union[FunctionNode, ArrayNode] = None + new_kwarg_flag = False + if len(candidates2) > 0: + # So that files(['a', 'b']) gets modified to files(['a', 'b', 'c']) instead of files(['a', 'b'], 'c') + if len({x for x in candidates2 if isinstance(x, ArrayNode)}) > 0: + candidates2 = {x for x in candidates2 if isinstance(x, ArrayNode)} + + # We choose one more or less arbitrary candidate + chosen = min(candidates2, key=lambda x: (x.lineno, x.colno)) + elif op == 'src_add': + chosen = target.node + elif op == 'extra_files_add': + chosen = ArrayNode(_symbol('['), ArgumentNode(Token('', tgt_function.filename, 0, 0, 0, None, '[]')), _symbol(']')) + + # this is fundamentally error prone + self.interpreter.dataflow_dag.add_edge(chosen, target.node) + + extra_files_idnode = IdNode(Token('string', tgt_function.filename, 0, 0, 0, None, 'extra_files')) + if tgt_function not in self.modified_nodes: + self.modified_nodes += [tgt_function] + new_extra_files_node: BaseNode + if target.node.args.get_kwarg_or_default('extra_files', None) is None: + # Target has no extra_files kwarg, create one + new_kwarg_flag = True + new_extra_files_node = chosen + else: + new_kwarg_flag = True + old_extra_files = target.node.args.get_kwarg_or_default('extra_files', None) + target.node.args.kwargs = {k: v for k, v in target.node.args.kwargs.items() if not (isinstance(k, IdNode) and k.value == 'extra_files')} + new_extra_files_node = ArithmeticNode('add', old_extra_files, _symbol('+'), chosen) + + tgt_function.args.kwargs[extra_files_idnode] = new_extra_files_node + + newfiles_relto = self.get_relto(target.node, chosen) + old_src_list: T.List[T.Any] = flatten([self.interpreter.node_to_runtime_value(sn) for sn in old]) + + if op == 'src_add': + name = 'Source' + elif op == 'extra_files_add': + name = 'Extra file' + # Generate the new String nodes + to_append = [] + added = [] + + old_src_list = [(target_dir_abs / x).resolve() if isinstance(x, str) else x.to_abs_path(source_root_abs) for x in old_src_list if not isinstance(x, UnknownValue)] + for _newf in sorted(set(newfiles)): + newf = Path(_newf) + if os.path.isabs(newf): + newf = Path(newf) + else: + newf = source_root_abs / newf + if newf in old_src_list: + mlog.log(' -- ', name, mlog.green(str(newf)), 'is already defined for the target --> skipping') + continue + + mlog.log(' -- Adding ', name.lower(), mlog.green(str(newf)), 'at', + mlog.yellow(f'{chosen.filename}:{chosen.lineno}')) + added.append(newf) + mocktarget = self.interpreter.funcvals[target.node] + assert isinstance(mocktarget, IntrospectionBuildTarget) + # print("adding ", str(newf), 'to', mocktarget.name) todo: should we write something to stderr? + + path = relpath(newf, newfiles_relto) + path = codecs.encode(path, 'unicode_escape').decode() # Because the StringNode constructor does the inverse + token = Token('string', chosen.filename, 0, 0, 0, None, path) + to_append += [StringNode(token)] + + assert isinstance(chosen, (FunctionNode, ArrayNode)) + arg_node = chosen.args + # Append to the AST at the right place + arg_node.arguments += to_append + + # Mark the node as modified + if chosen not in to_sort_nodes: + to_sort_nodes += [chosen] + # If the extra_files array is newly created, i.e. if new_kwarg_flag is + # True, don't mark it as its parent function node already is, otherwise + # this would cause double modification. + if chosen not in self.modified_nodes and not new_kwarg_flag: + self.modified_nodes += [chosen] + # Utility function to get a list of the sources from a node def arg_list_from_node(self, n: BaseNode) -> T.List[BaseNode]: args = [] @@ -646,18 +808,26 @@ class Rewriter: args = n.arguments return args - def rm_src_or_extra(self, op: str, target: IntrospectionBuildTarget, to_be_removed: T.List[str], to_sort_nodes: T.List[ArgumentNode]) -> None: + def rm_src_or_extra(self, op: str, target: IntrospectionBuildTarget, to_be_removed: T.List[str], to_sort_nodes: T.List[T.Union[FunctionNode, ArrayNode]]) -> None: + assert op in {'src_rm', 'extra_files_rm'} + cwd = Path(os.getcwd()) + source_root_abs = cwd / self.interpreter.source_root + # Helper to find the exact string node and its parent - def find_node(src: str) -> T.Tuple[BaseNode, StringNode]: + def find_node(src: str) -> T.Tuple[T.Optional[BaseNode], T.Optional[StringNode]]: if op == 'src_rm': - nodes = target.source_nodes + nodes = self.interpreter.dataflow_dag.reachable(set(target.source_nodes), True).union({target.node}) elif op == 'extra_files_rm': - nodes = target.extra_files + nodes = self.interpreter.dataflow_dag.reachable({target.extra_files}, True) for i in nodes: - for j in self.arg_list_from_node(i): - if isinstance(j, StringNode): - if j.value == src: - return i, j + if isinstance(i, UnknownValue): + continue + relto = self.get_relto(target.node, i) + if relto is not None: + for j in self.arg_list_from_node(i): + if isinstance(j, StringNode): + if os.path.normpath(relto / j.value) == os.path.normpath(source_root_abs / src): + return i, j return None, None if op == 'src_rm': @@ -671,22 +841,22 @@ class Rewriter: if root is None: mlog.warning(' -- Unable to find', name, mlog.green(i), 'in the target') continue + if not self.affects_no_other_targets(string_node): + mlog.warning(' -- Removing the', name, mlog.green(i), 'is too compilicated') + continue - arg_node = None - if isinstance(root, (FunctionNode, ArrayNode)): - arg_node = root.args - elif isinstance(root, ArgumentNode): - arg_node = root - assert arg_node is not None + if not isinstance(root, (FunctionNode, ArrayNode)): + raise NotImplementedError # I'm lazy # Remove the found string node from the argument list + arg_node = root.args mlog.log(' -- Removing', name, mlog.green(i), 'from', mlog.yellow(f'{string_node.filename}:{string_node.lineno}')) arg_node.arguments.remove(string_node) # Mark the node as modified - if arg_node not in to_sort_nodes and not isinstance(root, FunctionNode): - to_sort_nodes += [arg_node] + if root not in to_sort_nodes: + to_sort_nodes += [root] if root not in self.modified_nodes: self.modified_nodes += [root] @@ -711,106 +881,10 @@ class Rewriter: if target is not None: cmd['sources'] = [rel_source(x) for x in cmd['sources']] - to_sort_nodes = [] - - if cmd['operation'] == 'src_add': - node = None - if target.source_nodes: - node = target.source_nodes[0] - else: - node = target.node - assert node is not None - - # Generate the current source list - src_list = [] - for src_node in target.source_nodes: - for j in self.arg_list_from_node(src_node): - if isinstance(j, StringNode): - src_list += [j.value] - - # Generate the new String nodes - to_append = [] - for i in sorted(set(cmd['sources'])): - if i in src_list: - mlog.log(' -- Source', mlog.green(i), 'is already defined for the target --> skipping') - continue - mlog.log(' -- Adding source', mlog.green(i), 'at', - mlog.yellow(f'{node.filename}:{node.lineno}')) - token = Token('string', node.filename, 0, 0, 0, None, i) - to_append += [StringNode(token)] - - # Append to the AST at the right place - arg_node = None - if isinstance(node, (FunctionNode, ArrayNode)): - arg_node = node.args - elif isinstance(node, ArgumentNode): - arg_node = node - assert arg_node is not None - arg_node.arguments += to_append - - # Mark the node as modified - if arg_node not in to_sort_nodes and not isinstance(node, FunctionNode): - to_sort_nodes += [arg_node] - if node not in self.modified_nodes: - self.modified_nodes += [node] - - elif cmd['operation'] == 'extra_files_add': - tgt_function: FunctionNode = target.node - mark_array = True - try: - node = target.extra_files[0] - except IndexError: - # Specifying `extra_files` with a list that flattens to empty gives an empty - # target.extra_files list, account for that. - try: - extra_files_key = next(k for k in tgt_function.args.kwargs.keys() if isinstance(k, IdNode) and k.value == 'extra_files') - node = tgt_function.args.kwargs[extra_files_key] - except StopIteration: - # Target has no extra_files kwarg, create one - node = ArrayNode(_symbol('['), ArgumentNode(Token('', tgt_function.filename, 0, 0, 0, None, '[]')), _symbol(']')) - tgt_function.args.kwargs[IdNode(Token('string', tgt_function.filename, 0, 0, 0, None, 'extra_files'))] = node - mark_array = False - if tgt_function not in self.modified_nodes: - self.modified_nodes += [tgt_function] - target.extra_files = [node] - if isinstance(node, IdNode): - cv = self.interpreter.get_cur_value(node.value) - assert isinstance(cv, BaseNode) - node = cv - target.extra_files = [node] - if not isinstance(node, ArrayNode): - mlog.error('Target', mlog.bold(cmd['target']), 'extra_files argument must be a list', *self.on_error()) - return self.handle_error() - - # Generate the current extra files list - extra_files_list = [] - for i in target.extra_files: - for j in self.arg_list_from_node(i): - if isinstance(j, StringNode): - extra_files_list += [j.value] - - # Generate the new String nodes - to_append = [] - for i in sorted(set(cmd['sources'])): - if i in extra_files_list: - mlog.log(' -- Extra file', mlog.green(i), 'is already defined for the target --> skipping') - continue - mlog.log(' -- Adding extra file', mlog.green(i), 'at', - mlog.yellow(f'{node.filename}:{node.lineno}')) - token = Token('string', node.filename, 0, 0, 0, None, i) - to_append += [StringNode(token)] + to_sort_nodes: T.List[T.Union[FunctionNode, ArrayNode]] = [] - # Append to the AST at the right place - arg_node = node.args - arg_node.arguments += to_append - - # Mark the node as modified - if arg_node not in to_sort_nodes: - to_sort_nodes += [arg_node] - # If the extra_files array is newly created, don't mark it as its parent function node already is, - # otherwise this would cause double modification. - if mark_array and node not in self.modified_nodes: - self.modified_nodes += [node] + if cmd['operation'] in {'src_add', 'extra_files_add'}: + self.add_src_or_extra(cmd['operation'], target, cmd['sources'], to_sort_nodes) elif cmd['operation'] in {'src_rm', 'extra_files_rm'}: self.rm_src_or_extra(cmd['operation'], target, cmd['sources'], to_sort_nodes) @@ -823,7 +897,7 @@ class Rewriter: id_base = re.sub(r'[- ]', '_', cmd['target']) target_id = id_base + '_exe' if cmd['target_type'] == 'executable' else '_lib' source_id = id_base + '_sources' - filename = os.path.join(cmd['subdir'], environment.build_filename) + filename = os.path.join(os.getcwd(), self.interpreter.source_root, cmd['subdir'], environment.build_filename) # Build src list src_arg_node = ArgumentNode(Token('string', filename, 0, 0, 0, None, '')) @@ -857,16 +931,16 @@ class Rewriter: elif cmd['operation'] == 'info': # T.List all sources in the target - src_list = [] - for i in target.source_nodes: - for j in self.arg_list_from_node(i): - if isinstance(j, StringNode): - src_list += [j.value] - extra_files_list = [] - for i in target.extra_files: - for j in self.arg_list_from_node(i): - if isinstance(j, StringNode): - extra_files_list += [j.value] + + cwd = Path(os.getcwd()) + source_root_abs = cwd / self.interpreter.source_root + + src_list = self.interpreter.nodes_to_pretty_filelist(source_root_abs, target.subdir, target.source_nodes) + extra_files_list = self.interpreter.nodes_to_pretty_filelist(source_root_abs, target.subdir, [target.extra_files] if target.extra_files else []) + + src_list = ['unknown' if isinstance(x, UnknownValue) else relpath(x, source_root_abs) for x in src_list] + extra_files_list = ['unknown' if isinstance(x, UnknownValue) else relpath(x, source_root_abs) for x in extra_files_list] + test_data = { 'name': target.name, 'sources': src_list, @@ -885,10 +959,16 @@ class Rewriter: def path_sorter(key: str) -> T.List[T.Tuple[bool, T.List[T.Union[int, str]]]]: return [(key.count('/') <= idx, alphanum_key(x)) for idx, x in enumerate(key.split('/'))] - unknown = [x for x in i.arguments if not isinstance(x, StringNode)] - sources = [x for x in i.arguments if isinstance(x, StringNode)] + if isinstance(i, FunctionNode) and i.func_name.value in BUILD_TARGET_FUNCTIONS: + src_args = i.args.arguments[1:] + target_name = [i.args.arguments[0]] + else: + src_args = i.args.arguments + target_name = [] + unknown: T.List[BaseNode] = [x for x in src_args if not isinstance(x, StringNode)] + sources: T.List[StringNode] = [x for x in src_args if isinstance(x, StringNode)] sources = sorted(sources, key=lambda x: path_sorter(x.value)) - i.arguments = unknown + T.cast(T.List[BaseNode], sources) + i.args.arguments = target_name + unknown + T.cast(T.List[BaseNode], sources) def process(self, cmd: T.Dict[str, T.Any]) -> None: if 'type' not in cmd: @@ -931,7 +1011,7 @@ class Rewriter: for i in str_list: if i['file'] in files: continue - fpath = os.path.realpath(os.path.join(self.sourcedir, T.cast(str, i['file']))) + fpath = os.path.realpath(T.cast(str, i['file'])) fdata = '' # Create an empty file if it does not exist if not os.path.exists(fpath): @@ -1077,12 +1157,22 @@ def run(options: argparse.Namespace) -> int: if not isinstance(commands, list): raise TypeError('Command is not a list') - for i in commands: - if not isinstance(i, object): + for i, cmd in enumerate(commands): + if not isinstance(cmd, object): raise TypeError('Command is not an object') - rewriter.process(i) + rewriter.process(cmd) + rewriter.apply_changes() + + if i == len(commands) - 1: # Improves the performance, is not necessary for correctness. + break + + rewriter.modified_nodes = [] + rewriter.to_remove_nodes = [] + rewriter.to_add_nodes = [] + # The AST changed, so we need to update every information that was derived from the AST + rewriter.interpreter = IntrospectionInterpreter(rewriter.sourcedir, '', rewriter.interpreter.backend, visitors = [AstIDGenerator(), AstIndentationGenerator(), AstConditionLevel()]) + rewriter.analyze_meson() - rewriter.apply_changes() rewriter.print_info() return 0 except Exception as e: diff --git a/test cases/rewrite/1 basic/addSrc.json b/test cases/rewrite/1 basic/addSrc.json index b8bc43916..52603f60b 100644 --- a/test cases/rewrite/1 basic/addSrc.json +++ b/test cases/rewrite/1 basic/addSrc.json @@ -43,6 +43,24 @@ }, { "type": "target", + "target": "trivialprog10", + "operation": "src_add", + "sources": ["fileA.cpp", "fileB.cpp", "a1.cpp"] + }, + { + "type": "target", + "target": "trivialprog11", + "operation": "src_add", + "sources": ["fileA.cpp", "a1.cpp"] + }, + { + "type": "target", + "target": "trivialprog12", + "operation": "src_add", + "sources": ["fileA.cpp", "fileB.cpp", "a1.cpp"] + }, + { + "type": "target", "target": "trivialprog0", "operation": "info" }, @@ -90,5 +108,25 @@ "type": "target", "target": "trivialprog9", "operation": "info" + }, + { + "type": "target", + "target": "trivialprog10", + "operation": "info" + }, + { + "type": "target", + "target": "trivialprog11", + "operation": "info" + }, + { + "type": "target", + "target": "trivialprog12", + "operation": "info" + }, + { + "type": "target", + "target": "rightName", + "operation": "info" } ] diff --git a/test cases/rewrite/1 basic/addTgt.json b/test cases/rewrite/1 basic/addTgt.json index 2f4e7e256..02d600a82 100644 --- a/test cases/rewrite/1 basic/addTgt.json +++ b/test cases/rewrite/1 basic/addTgt.json @@ -1,7 +1,7 @@ [ { "type": "target", - "target": "trivialprog10", + "target": "trivialprog13", "operation": "target_add", "sources": ["new1.cpp", "new2.cpp"], "target_type": "shared_library" diff --git a/test cases/rewrite/1 basic/expected_dag.txt b/test cases/rewrite/1 basic/expected_dag.txt index abc04fc31..c5025b4ad 100644 --- a/test cases/rewrite/1 basic/expected_dag.txt +++ b/test cases/rewrite/1 basic/expected_dag.txt @@ -15,69 +15,115 @@ Data flowing to ArrayNode(6:7): IdNode(6:8) Data flowing to IdNode(6:8): IdNode(5:7) +Data flowing to ArithmeticNode(7:7): + ArrayNode(7:7) + ArrayNode(8:8) +Data flowing to ArrayNode(7:7): + StringNode(7:8) + StringNode(7:20) +Data flowing to ArrayNode(8:8): + StringNode(8:9) +Data flowing to ArrayNode(9:7): + StringNode(9:8) + StringNode(9:20) Data flowing to FunctionNode(10:7): - StringNode(10:18) - ArithmeticNode(10:34) -Data flowing to ArithmeticNode(10:34): - IdNode(10:34) - IdNode(10:41) -Data flowing to IdNode(10:34): + IdNode(10:13) +Data flowing to IdNode(10:13): + ArrayNode(9:7) +Data flowing to ArrayNode(11:7): + StringNode(11:8) + StringNode(11:20) +Data flowing to IdNode(12:7): + ArrayNode(11:7) +Data flowing to ArrayNode(13:7): + StringNode(13:8) + StringNode(13:21) +Data flowing to FunctionNode(15:13): + StringNode(14:7) + StringNode(15:26) +Data flowing to FunctionNode(20:7): + StringNode(20:18) + ArithmeticNode(20:34) +Data flowing to ArithmeticNode(20:34): + IdNode(20:34) + IdNode(20:41) +Data flowing to IdNode(20:34): ArrayNode(3:7) -Data flowing to IdNode(10:41): +Data flowing to IdNode(20:41): FunctionNode(4:7) -Data flowing to FunctionNode(11:7): - StringNode(11:18) - IdNode(11:34) -Data flowing to IdNode(11:34): +Data flowing to FunctionNode(21:7): + StringNode(21:18) + IdNode(21:34) +Data flowing to IdNode(21:34): ArrayNode(3:7) -Data flowing to FunctionNode(12:7): - StringNode(12:18) - ArrayNode(12:34) -Data flowing to ArrayNode(12:34): - IdNode(12:35) -Data flowing to IdNode(12:35): +Data flowing to FunctionNode(22:7): + StringNode(22:18) + ArrayNode(22:34) +Data flowing to ArrayNode(22:34): + IdNode(22:35) +Data flowing to IdNode(22:35): FunctionNode(4:7) -Data flowing to FunctionNode(13:7): - StringNode(13:18) - ArrayNode(13:34) -Data flowing to ArrayNode(13:34): - StringNode(13:35) - StringNode(13:47) -Data flowing to FunctionNode(14:7): - StringNode(14:18) - ArrayNode(14:34) -Data flowing to ArrayNode(14:34): - StringNode(14:35) - ArrayNode(14:47) -Data flowing to ArrayNode(14:47): - StringNode(14:48) -Data flowing to FunctionNode(15:7): - StringNode(15:18) - ArrayNode(15:34) -Data flowing to ArrayNode(15:34): - IdNode(15:35) - StringNode(15:41) -Data flowing to IdNode(15:35): +Data flowing to FunctionNode(23:7): + StringNode(23:18) + ArrayNode(23:34) +Data flowing to ArrayNode(23:34): + StringNode(23:35) + StringNode(23:47) +Data flowing to FunctionNode(24:7): + StringNode(24:18) + ArrayNode(24:34) +Data flowing to ArrayNode(24:34): + StringNode(24:35) + ArrayNode(24:47) +Data flowing to ArrayNode(24:47): + StringNode(24:48) +Data flowing to FunctionNode(25:7): + StringNode(25:18) + ArrayNode(25:34) +Data flowing to ArrayNode(25:34): + IdNode(25:35) + StringNode(25:41) +Data flowing to IdNode(25:35): FunctionNode(4:7) -Data flowing to FunctionNode(16:7): - StringNode(16:18) - StringNode(16:34) - StringNode(16:46) -Data flowing to FunctionNode(17:7): - StringNode(17:18) - StringNode(17:34) - IdNode(17:47) - StringNode(17:53) -Data flowing to IdNode(17:47): +Data flowing to FunctionNode(26:7): + StringNode(26:18) + StringNode(26:34) + StringNode(26:46) +Data flowing to FunctionNode(27:7): + StringNode(27:18) + StringNode(27:34) + FunctionNode(27:47) + StringNode(27:69) +Data flowing to FunctionNode(27:47): ArrayNode(3:7) -Data flowing to FunctionNode(18:7): - StringNode(18:18) - IdNode(18:34) -Data flowing to IdNode(18:34): + StringNode(27:60) +Data flowing to FunctionNode(28:7): + StringNode(28:18) + IdNode(28:34) +Data flowing to IdNode(28:34): IdNode(5:7) -Data flowing to FunctionNode(19:0): - StringNode(19:11) - IdNode(19:27) -Data flowing to IdNode(19:27): +Data flowing to FunctionNode(29:0): + StringNode(29:11) + IdNode(29:27) +Data flowing to IdNode(29:27): ArrayNode(6:7) - +Data flowing to FunctionNode(30:0): + StringNode(30:11) + IdNode(30:28) +Data flowing to IdNode(30:28): + ArithmeticNode(7:7) +Data flowing to FunctionNode(31:0): + StringNode(31:11) + IdNode(31:28) +Data flowing to IdNode(31:28): + FunctionNode(10:7) +Data flowing to FunctionNode(32:0): + StringNode(32:11) + IdNode(32:28) +Data flowing to IdNode(32:28): + IdNode(12:7) +Data flowing to FunctionNode(33:0): + IdNode(33:11) + StringNode(33:23) +Data flowing to IdNode(33:11): + FunctionNode(15:13) diff --git a/test cases/rewrite/1 basic/info.json b/test cases/rewrite/1 basic/info.json index 0f1a3bd8c..9977f5a87 100644 --- a/test cases/rewrite/1 basic/info.json +++ b/test cases/rewrite/1 basic/info.json @@ -53,5 +53,25 @@ "type": "target", "target": "trivialprog10", "operation": "info" + }, + { + "type": "target", + "target": "trivialprog11", + "operation": "info" + }, + { + "type": "target", + "target": "trivialprog12", + "operation": "info" + }, + { + "type": "target", + "target": "trivialprog13", + "operation": "info" + }, + { + "type": "target", + "target": "rightName", + "operation": "info" } ] diff --git a/test cases/rewrite/1 basic/meson.build b/test cases/rewrite/1 basic/meson.build index 0f87c4520..5fe952769 100644 --- a/test cases/rewrite/1 basic/meson.build +++ b/test cases/rewrite/1 basic/meson.build @@ -4,6 +4,16 @@ src1 = ['main.cpp', 'fileA.cpp'] src2 = files(['fileB.cpp', 'fileC.cpp']) src3 = src1 src4 = [src3] +src5 = ['main.cpp', 'fileA.cpp'] +src5 += ['fileB.cpp'] +src6 = ['main.cpp', 'fileA.cpp'] +src6 = files(src6) +src7 = ['main.cpp', 'fileA.cpp'] +src8 = src7 +src7 = ['fileB.cpp', 'fileC.cpp'] +name = 'rightName' +trickyName = get_variable('name') +name = 'wrongName' # Magic comment @@ -14,6 +24,10 @@ exe3 = executable('trivialprog3', ['main.cpp', 'fileA.cpp']) exe4 = executable('trivialprog4', ['main.cpp', ['fileA.cpp']]) exe5 = executable('trivialprog5', [src2, 'main.cpp']) exe6 = executable('trivialprog6', 'main.cpp', 'fileA.cpp') -exe7 = executable('trivialprog7', 'fileB.cpp', src1, 'fileC.cpp') +exe7 = executable('trivialprog7', 'fileB.cpp', get_variable('src1'), 'fileC.cpp') exe8 = executable('trivialprog8', src3) executable('trivialprog9', src4) +executable('trivialprog10', src5) +executable('trivialprog11', src6) +executable('trivialprog12', src8) +executable(trickyName, 'main.cpp') diff --git a/test cases/rewrite/1 basic/rmSrc.json b/test cases/rewrite/1 basic/rmSrc.json index 2e7447c68..de56bbe1b 100644 --- a/test cases/rewrite/1 basic/rmSrc.json +++ b/test cases/rewrite/1 basic/rmSrc.json @@ -1,12 +1,6 @@ [ { "type": "target", - "target": "trivialprog1", - "operation": "src_rm", - "sources": ["fileA.cpp"] - }, - { - "type": "target", "target": "trivialprog3", "operation": "src_rm", "sources": ["fileA.cpp"] @@ -21,7 +15,7 @@ "type": "target", "target": "trivialprog5", "operation": "src_rm", - "sources": ["fileB.cpp"] + "sources": ["main.cpp"] }, { "type": "target", @@ -37,6 +31,18 @@ }, { "type": "target", + "target": "trivialprog10", + "operation": "src_rm", + "sources": ["fileA.cpp", "fileB.cpp"] + }, + { + "type": "target", + "target": "trivialprog11", + "operation": "src_rm", + "sources": ["fileA.cpp"] + }, + { + "type": "target", "target": "trivialprog0", "operation": "info" }, @@ -84,5 +90,25 @@ "type": "target", "target": "trivialprog9", "operation": "info" + }, + { + "type": "target", + "target": "trivialprog10", + "operation": "info" + }, + { + "type": "target", + "target": "trivialprog11", + "operation": "info" + }, + { + "type": "target", + "target": "trivialprog12", + "operation": "info" + }, + { + "type": "target", + "target": "rightName", + "operation": "info" } ] diff --git a/test cases/rewrite/1 basic/rmTgt.json b/test cases/rewrite/1 basic/rmTgt.json index dbaf02535..bc6dc302e 100644 --- a/test cases/rewrite/1 basic/rmTgt.json +++ b/test cases/rewrite/1 basic/rmTgt.json @@ -13,5 +13,10 @@ "type": "target", "target": "trivialprog9", "operation": "target_rm" + }, + { + "type": "target", + "target": "rightName", + "operation": "target_rm" } ] diff --git a/test cases/rewrite/7 tricky dataflow/addSrc.json b/test cases/rewrite/7 tricky dataflow/addSrc.json new file mode 100644 index 000000000..1b6aa485b --- /dev/null +++ b/test cases/rewrite/7 tricky dataflow/addSrc.json @@ -0,0 +1,72 @@ +[ + { + "type": "target", + "target": "tgt1", + "operation": "src_add", + "sources": [ + "new.c" + ] + }, + { + "type": "target", + "target": "tgt2", + "operation": "src_add", + "sources": [ + "new.c" + ] + }, + { + "type": "target", + "target": "tgt3", + "operation": "src_add", + "sources": [ + "new.c" + ] + }, + { + "type": "target", + "target": "tgt5", + "operation": "src_add", + "sources": [ + "new.c" + ] + }, + { + "type": "target", + "target": "tgt6", + "operation": "src_add", + "sources": [ + "new.c" + ] + }, + { + "type": "target", + "target": "tgt1", + "operation": "info" + }, + { + "type": "target", + "target": "tgt2", + "operation": "info" + }, + { + "type": "target", + "target": "tgt3", + "operation": "info" + }, + { + "type": "target", + "target": "tgt4", + "operation": "info" + }, + { + "type": "target", + "target": "tgt5", + "operation": "info" + }, + { + "type": "target", + "target": "tgt6", + "operation": "info" + } +] diff --git a/test cases/rewrite/7 tricky dataflow/info.json b/test cases/rewrite/7 tricky dataflow/info.json new file mode 100644 index 000000000..5fc65dd8c --- /dev/null +++ b/test cases/rewrite/7 tricky dataflow/info.json @@ -0,0 +1,32 @@ +[ + { + "type": "target", + "target": "tgt1", + "operation": "info" + }, + { + "type": "target", + "target": "tgt2", + "operation": "info" + }, + { + "type": "target", + "target": "tgt3", + "operation": "info" + }, + { + "type": "target", + "target": "tgt4", + "operation": "info" + }, + { + "type": "target", + "target": "tgt5", + "operation": "info" + }, + { + "type": "target", + "target": "tgt6", + "operation": "info" + } +] diff --git a/test cases/rewrite/7 tricky dataflow/meson.build b/test cases/rewrite/7 tricky dataflow/meson.build new file mode 100644 index 000000000..4c183ed97 --- /dev/null +++ b/test cases/rewrite/7 tricky dataflow/meson.build @@ -0,0 +1,35 @@ +project('rewrite tricky dataflow', 'c') + +# Adding a file to `begin` will add this file to the sources of `tgt1`, but +# not to any other target. But a buggy rewriter might think that adding a file +# to `begin` will also add this file to `end` and will refuse to add a file to +# `begin`. +begin = ['foo.c'] +tgt1 = library('tgt1', begin) +distraction = executable('distraction', link_with: tgt1) + + +tgt2_srcs = ['foo.c'] +if meson.host_machine().system() == 'windows' # Some condition that cannot be known statically + tgt2_srcs += ['bar.c'] +endif +executable('tgt2', tgt2_srcs) + + +tgt34_srcs = ['foo.c'] +executable('tgt3', tgt34_srcs) +if meson.host_machine().system() == 'windows' + tgt34_srcs += ['bar.c'] +endif +executable('tgt4', tgt34_srcs) + + +dont_add_here_5 = ['foo.c'] +ct = custom_target('ct', output: 'out.c', input: dont_add_here_5, command: ['placeholder', '@INPUT@', '@OUTPUT@']) +executable('tgt5', ct) + + +dont_add_here_6 = ['foo.c'] +gen = generator(find_program('cp'), output: '@BASENAME@_copy.c', arguments: ['@INPUT@', '@OUTPUT@']) +generated = gen.process(dont_add_here_6) +executable('tgt6', generated) diff --git a/unittests/allplatformstests.py b/unittests/allplatformstests.py index b983f341c..fa6f11965 100644 --- a/unittests/allplatformstests.py +++ b/unittests/allplatformstests.py @@ -3612,6 +3612,8 @@ class AllPlatformTests(BasePlatformTests): # Account for differences in output res_wb = [i for i in res_wb if i['type'] != 'custom'] for i in res_wb: + if i['id'] == 'test1@exe': + i['build_by_default'] = 'unknown' i['filename'] = [os.path.relpath(x, self.builddir) for x in i['filename']] for k in ('install_filename', 'dependencies', 'win_subsystem'): if k in i: @@ -3730,7 +3732,7 @@ class AllPlatformTests(BasePlatformTests): }, { 'name': 'bugDep1', - 'required': True, + 'required': 'unknown', 'version': [], 'has_fallback': False, 'conditional': False diff --git a/unittests/rewritetests.py b/unittests/rewritetests.py index 70a80347e..75405efc8 100644 --- a/unittests/rewritetests.py +++ b/unittests/rewritetests.py @@ -73,6 +73,10 @@ class RewriterTests(BasePlatformTests): 'trivialprog7@exe': {'name': 'trivialprog7', 'sources': ['fileB.cpp', 'fileC.cpp', 'main.cpp', 'fileA.cpp'], 'extra_files': []}, 'trivialprog8@exe': {'name': 'trivialprog8', 'sources': ['main.cpp', 'fileA.cpp'], 'extra_files': []}, 'trivialprog9@exe': {'name': 'trivialprog9', 'sources': ['main.cpp', 'fileA.cpp'], 'extra_files': []}, + 'trivialprog10@exe': {'name': 'trivialprog10', 'sources': ['main.cpp', 'fileA.cpp', 'fileB.cpp'], 'extra_files': []}, + 'trivialprog11@exe': {'name': 'trivialprog11', 'sources': ['main.cpp', 'fileA.cpp'], 'extra_files': []}, + 'trivialprog12@exe': {'name': 'trivialprog12', 'sources': ['main.cpp', 'fileA.cpp'], 'extra_files': []}, + 'rightName@exe': {'name': 'rightName', 'sources': ['main.cpp'], 'extra_files': []}, } } self.assertEqualIgnoreOrder(out, expected) @@ -82,16 +86,20 @@ class RewriterTests(BasePlatformTests): out = self.rewrite(self.builddir, os.path.join(self.builddir, 'addSrc.json')) expected = { 'target': { - 'trivialprog0@exe': {'name': 'trivialprog0', 'sources': ['a1.cpp', 'a2.cpp', 'a6.cpp', 'fileA.cpp', 'main.cpp', 'a7.cpp', 'fileB.cpp', 'fileC.cpp'], 'extra_files': []}, - 'trivialprog1@exe': {'name': 'trivialprog1', 'sources': ['a1.cpp', 'a2.cpp', 'a6.cpp', 'fileA.cpp', 'main.cpp'], 'extra_files': []}, - 'trivialprog2@exe': {'name': 'trivialprog2', 'sources': ['a7.cpp', 'fileB.cpp', 'fileC.cpp'], 'extra_files': []}, + 'trivialprog0@exe': {'name': 'trivialprog0', 'sources': ['main.cpp', 'fileA.cpp', 'fileB.cpp', 'fileC.cpp'], 'extra_files': []}, + 'trivialprog1@exe': {'name': 'trivialprog1', 'sources': ['main.cpp', 'fileA.cpp', 'a1.cpp', 'a2.cpp'], 'extra_files': []}, + 'trivialprog2@exe': {'name': 'trivialprog2', 'sources': ['fileB.cpp', 'fileC.cpp', 'a7.cpp'], 'extra_files': []}, 'trivialprog3@exe': {'name': 'trivialprog3', 'sources': ['a5.cpp', 'fileA.cpp', 'main.cpp'], 'extra_files': []}, - 'trivialprog4@exe': {'name': 'trivialprog4', 'sources': ['a5.cpp', 'main.cpp', 'fileA.cpp'], 'extra_files': []}, - 'trivialprog5@exe': {'name': 'trivialprog5', 'sources': ['a3.cpp', 'main.cpp', 'a7.cpp', 'fileB.cpp', 'fileC.cpp'], 'extra_files': []}, + 'trivialprog4@exe': {'name': 'trivialprog4', 'sources': ['main.cpp', 'a5.cpp', 'fileA.cpp'], 'extra_files': []}, + 'trivialprog5@exe': {'name': 'trivialprog5', 'sources': ['fileB.cpp', 'fileC.cpp', 'a3.cpp', 'main.cpp'], 'extra_files': []}, 'trivialprog6@exe': {'name': 'trivialprog6', 'sources': ['main.cpp', 'fileA.cpp', 'a4.cpp'], 'extra_files': []}, - 'trivialprog7@exe': {'name': 'trivialprog7', 'sources': ['fileB.cpp', 'fileC.cpp', 'a1.cpp', 'a2.cpp', 'a6.cpp', 'fileA.cpp', 'main.cpp'], 'extra_files': []}, - 'trivialprog8@exe': {'name': 'trivialprog8', 'sources': ['a1.cpp', 'a2.cpp', 'a6.cpp', 'fileA.cpp', 'main.cpp'], 'extra_files': []}, - 'trivialprog9@exe': {'name': 'trivialprog9', 'sources': ['a1.cpp', 'a2.cpp', 'a6.cpp', 'fileA.cpp', 'main.cpp'], 'extra_files': []}, + 'trivialprog7@exe': {'name': 'trivialprog7', 'sources': ['fileB.cpp', 'fileC.cpp', 'fileA.cpp', 'main.cpp'], 'extra_files': []}, + 'trivialprog8@exe': {'name': 'trivialprog8', 'sources': ['main.cpp', 'fileA.cpp'], 'extra_files': []}, + 'trivialprog9@exe': {'name': 'trivialprog9', 'sources': ['main.cpp', 'fileA.cpp', 'a1.cpp', 'a6.cpp' ], 'extra_files': []}, + 'trivialprog10@exe': {'name': 'trivialprog10', 'sources': ['main.cpp', 'fileA.cpp', 'fileB.cpp', 'a1.cpp'], 'extra_files': []}, + 'trivialprog11@exe': {'name': 'trivialprog11', 'sources': ['a1.cpp', 'fileA.cpp', 'main.cpp'], 'extra_files': []}, + 'trivialprog12@exe': {'name': 'trivialprog12', 'sources': ['a1.cpp', 'fileA.cpp', 'fileB.cpp', 'main.cpp'], 'extra_files': []}, + 'rightName@exe': {'name': 'rightName', 'sources': ['main.cpp'], 'extra_files': []}, } } self.assertEqualIgnoreOrder(out, expected) @@ -107,7 +115,7 @@ class RewriterTests(BasePlatformTests): inf = json.dumps([{"type": "target", "target": "trivialprog1", "operation": "info"}]) self.rewrite(self.builddir, add) out = self.rewrite(self.builddir, inf) - expected = {'target': {'trivialprog1@exe': {'name': 'trivialprog1', 'sources': ['a1.cpp', 'a2.cpp', 'a6.cpp', 'fileA.cpp', 'main.cpp'], 'extra_files': []}}} + expected = {'target': {'trivialprog1@exe': {'name': 'trivialprog1', 'sources': ['main.cpp', 'fileA.cpp', 'a1.cpp', 'a2.cpp', 'a6.cpp'], 'extra_files': []}}} self.assertDictEqual(out, expected) def test_target_remove_sources(self): @@ -115,16 +123,20 @@ class RewriterTests(BasePlatformTests): out = self.rewrite(self.builddir, os.path.join(self.builddir, 'rmSrc.json')) expected = { 'target': { - 'trivialprog0@exe': {'name': 'trivialprog0', 'sources': ['main.cpp', 'fileC.cpp'], 'extra_files': []}, - 'trivialprog1@exe': {'name': 'trivialprog1', 'sources': ['main.cpp'], 'extra_files': []}, - 'trivialprog2@exe': {'name': 'trivialprog2', 'sources': ['fileC.cpp'], 'extra_files': []}, + 'trivialprog0@exe': {'name': 'trivialprog0', 'sources': ['main.cpp', 'fileA.cpp', 'fileB.cpp', 'fileC.cpp'], 'extra_files': []}, + 'trivialprog1@exe': {'name': 'trivialprog1', 'sources': ['main.cpp', 'fileA.cpp'], 'extra_files': []}, + 'trivialprog2@exe': {'name': 'trivialprog2', 'sources': ['fileB.cpp', 'fileC.cpp'], 'extra_files': []}, 'trivialprog3@exe': {'name': 'trivialprog3', 'sources': ['main.cpp'], 'extra_files': []}, 'trivialprog4@exe': {'name': 'trivialprog4', 'sources': ['main.cpp'], 'extra_files': []}, - 'trivialprog5@exe': {'name': 'trivialprog5', 'sources': ['main.cpp', 'fileC.cpp'], 'extra_files': []}, + 'trivialprog5@exe': {'name': 'trivialprog5', 'sources': ['fileB.cpp', 'fileC.cpp'], 'extra_files': []}, 'trivialprog6@exe': {'name': 'trivialprog6', 'sources': ['main.cpp'], 'extra_files': []}, - 'trivialprog7@exe': {'name': 'trivialprog7', 'sources': ['fileC.cpp', 'main.cpp'], 'extra_files': []}, - 'trivialprog8@exe': {'name': 'trivialprog8', 'sources': ['main.cpp'], 'extra_files': []}, - 'trivialprog9@exe': {'name': 'trivialprog9', 'sources': ['main.cpp'], 'extra_files': []}, + 'trivialprog7@exe': {'name': 'trivialprog7', 'sources': ['fileC.cpp', 'main.cpp', 'fileA.cpp'], 'extra_files': []}, + 'trivialprog8@exe': {'name': 'trivialprog8', 'sources': ['main.cpp', 'fileA.cpp'], 'extra_files': []}, + 'trivialprog9@exe': {'name': 'trivialprog9', 'sources': ['main.cpp', 'fileA.cpp'], 'extra_files': []}, + 'trivialprog10@exe': {'name': 'trivialprog10', 'sources': ['main.cpp'], 'extra_files': []}, + 'trivialprog11@exe': {'name': 'trivialprog11', 'sources': ['main.cpp'], 'extra_files': []}, + 'trivialprog12@exe': {'name': 'trivialprog12', 'sources': ['main.cpp', 'fileA.cpp'], 'extra_files': []}, + 'rightName@exe': {'name': 'rightName', 'sources': ['main.cpp'], 'extra_files': []}, } } self.assertEqualIgnoreOrder(out, expected) @@ -136,7 +148,7 @@ class RewriterTests(BasePlatformTests): def test_target_subdir(self): self.prime('2 subdirs') out = self.rewrite(self.builddir, os.path.join(self.builddir, 'addSrc.json')) - expected = {'name': 'something', 'sources': ['first.c', 'second.c', 'third.c'], 'extra_files': []} + expected = {'name': 'something', 'sources': ['third.c', f'sub2{os.path.sep}first.c', f'sub2{os.path.sep}second.c'], 'extra_files': []} self.assertDictEqual(list(out['target'].values())[0], expected) # Check the written file @@ -157,6 +169,9 @@ class RewriterTests(BasePlatformTests): 'trivialprog6@exe': {'name': 'trivialprog6', 'sources': ['main.cpp', 'fileA.cpp'], 'extra_files': []}, 'trivialprog7@exe': {'name': 'trivialprog7', 'sources': ['fileB.cpp', 'fileC.cpp', 'main.cpp', 'fileA.cpp'], 'extra_files': []}, 'trivialprog8@exe': {'name': 'trivialprog8', 'sources': ['main.cpp', 'fileA.cpp'], 'extra_files': []}, + 'trivialprog10@exe': {'name': 'trivialprog10', 'sources': ['main.cpp', 'fileA.cpp', 'fileB.cpp'], 'extra_files': []}, + 'trivialprog11@exe': {'name': 'trivialprog11', 'sources': ['main.cpp', 'fileA.cpp'], 'extra_files': []}, + 'trivialprog12@exe': {'name': 'trivialprog12', 'sources': ['main.cpp', 'fileA.cpp'], 'extra_files': []}, } } self.assertEqualIgnoreOrder(out, expected) @@ -178,7 +193,11 @@ class RewriterTests(BasePlatformTests): 'trivialprog7@exe': {'name': 'trivialprog7', 'sources': ['fileB.cpp', 'fileC.cpp', 'main.cpp', 'fileA.cpp'], 'extra_files': []}, 'trivialprog8@exe': {'name': 'trivialprog8', 'sources': ['main.cpp', 'fileA.cpp'], 'extra_files': []}, 'trivialprog9@exe': {'name': 'trivialprog9', 'sources': ['main.cpp', 'fileA.cpp'], 'extra_files': []}, - 'trivialprog10@sha': {'name': 'trivialprog10', 'sources': ['new1.cpp', 'new2.cpp'], 'extra_files': []}, + 'trivialprog10@exe': {'name': 'trivialprog10', 'sources': ['main.cpp', 'fileA.cpp', 'fileB.cpp'], 'extra_files': []}, + 'trivialprog11@exe': {'name': 'trivialprog11', 'sources': ['main.cpp', 'fileA.cpp'], 'extra_files': []}, + 'trivialprog12@exe': {'name': 'trivialprog12', 'sources': ['main.cpp', 'fileA.cpp'], 'extra_files': []}, + 'trivialprog13@sha': {'name': 'trivialprog13', 'sources': ['new1.cpp', 'new2.cpp'], 'extra_files': []}, + 'rightName@exe': {'name': 'rightName', 'sources': ['main.cpp'], 'extra_files': []}, } } self.assertEqualIgnoreOrder(out, expected) @@ -193,7 +212,7 @@ class RewriterTests(BasePlatformTests): self.prime('2 subdirs') self.rewrite(self.builddir, os.path.join(self.builddir, 'addTgt.json')) out = self.rewrite(self.builddir, os.path.join(self.builddir, 'info.json')) - expected = {'name': 'something', 'sources': ['first.c', 'second.c'], 'extra_files': []} + expected = {'name': 'something', 'sources': [f'sub2{os.path.sep}first.c', f'sub2{os.path.sep}second.c'], 'extra_files': []} self.assertDictEqual(out['target']['94b671c@@something@exe'], expected) def test_target_source_sorting(self): @@ -240,16 +259,23 @@ class RewriterTests(BasePlatformTests): } } } + for k1, v1 in expected.items(): + for k2, v2 in v1.items(): + for k3, v3 in v2.items(): + if isinstance(v3, list): + for i in range(len(v3)): + v3[i] = v3[i].replace('/', os.path.sep) self.assertDictEqual(out, expected) def test_target_same_name_skip(self): self.prime('4 same name targets') out = self.rewrite(self.builddir, os.path.join(self.builddir, 'addSrc.json')) out = self.rewrite(self.builddir, os.path.join(self.builddir, 'info.json')) - expected = {'name': 'myExe', 'sources': ['main.cpp'], 'extra_files': []} + expected1 = {'name': 'myExe', 'sources': ['main.cpp'], 'extra_files': []} + expected2 = {'name': 'myExe', 'sources': [f'sub1{os.path.sep}main.cpp'], 'extra_files': []} self.assertEqual(len(out['target']), 2) - for val in out['target'].values(): - self.assertDictEqual(expected, val) + self.assertDictEqual(expected1, out['target']['myExe@exe']) + self.assertDictEqual(expected2, out['target']['9a11041@@myExe@exe']) def test_kwargs_info(self): self.prime('3 kwargs') @@ -359,14 +385,14 @@ class RewriterTests(BasePlatformTests): out = self.rewrite(self.builddir, os.path.join(self.builddir, 'addExtraFiles.json')) expected = { 'target': { - 'trivialprog0@exe': {'name': 'trivialprog0', 'sources': ['main.cpp'], 'extra_files': ['a1.hpp', 'a2.hpp', 'a6.hpp', 'fileA.hpp', 'main.hpp', 'a7.hpp', 'fileB.hpp', 'fileC.hpp']}, - 'trivialprog1@exe': {'name': 'trivialprog1', 'sources': ['main.cpp'], 'extra_files': ['a1.hpp', 'a2.hpp', 'a6.hpp', 'fileA.hpp', 'main.hpp']}, + 'trivialprog0@exe': {'name': 'trivialprog0', 'sources': ['main.cpp'], 'extra_files': ['fileA.hpp', 'main.hpp', 'fileB.hpp', 'fileC.hpp']}, + 'trivialprog1@exe': {'name': 'trivialprog1', 'sources': ['main.cpp'], 'extra_files': ['a1.hpp', 'a2.hpp', 'fileA.hpp', 'main.hpp']}, 'trivialprog2@exe': {'name': 'trivialprog2', 'sources': ['main.cpp'], 'extra_files': ['a7.hpp', 'fileB.hpp', 'fileC.hpp']}, 'trivialprog3@exe': {'name': 'trivialprog3', 'sources': ['main.cpp'], 'extra_files': ['a5.hpp', 'fileA.hpp', 'main.hpp']}, 'trivialprog4@exe': {'name': 'trivialprog4', 'sources': ['main.cpp'], 'extra_files': ['a5.hpp', 'main.hpp', 'fileA.hpp']}, - 'trivialprog5@exe': {'name': 'trivialprog5', 'sources': ['main.cpp'], 'extra_files': ['a3.hpp', 'main.hpp', 'a7.hpp', 'fileB.hpp', 'fileC.hpp']}, - 'trivialprog6@exe': {'name': 'trivialprog6', 'sources': ['main.cpp'], 'extra_files': ['a1.hpp', 'a2.hpp', 'a6.hpp', 'fileA.hpp', 'main.hpp']}, - 'trivialprog7@exe': {'name': 'trivialprog7', 'sources': ['main.cpp'], 'extra_files': ['a1.hpp', 'a2.hpp', 'a6.hpp', 'fileA.hpp', 'main.hpp']}, + 'trivialprog5@exe': {'name': 'trivialprog5', 'sources': ['main.cpp'], 'extra_files': ['a3.hpp', 'main.hpp', 'fileB.hpp', 'fileC.hpp']}, + 'trivialprog6@exe': {'name': 'trivialprog6', 'sources': ['main.cpp'], 'extra_files': ['fileA.hpp', 'main.hpp']}, + 'trivialprog7@exe': {'name': 'trivialprog7', 'sources': ['main.cpp'], 'extra_files': ['a1.hpp', 'a6.hpp', 'fileA.hpp', 'main.hpp']}, 'trivialprog8@exe': {'name': 'trivialprog8', 'sources': ['main.cpp'], 'extra_files': ['a2.hpp', 'a7.hpp']}, 'trivialprog9@exe': {'name': 'trivialprog9', 'sources': ['main.cpp'], 'extra_files': ['a8.hpp', 'a9.hpp']}, 'trivialprog10@exe': {'name': 'trivialprog10', 'sources': ['main.cpp'], 'extra_files': ['a1.hpp', 'a4.hpp']}, @@ -383,14 +409,14 @@ class RewriterTests(BasePlatformTests): out = self.rewrite(self.builddir, os.path.join(self.builddir, 'rmExtraFiles.json')) expected = { 'target': { - 'trivialprog0@exe': {'name': 'trivialprog0', 'sources': ['main.cpp'], 'extra_files': ['main.hpp', 'fileC.hpp']}, - 'trivialprog1@exe': {'name': 'trivialprog1', 'sources': ['main.cpp'], 'extra_files': ['main.hpp']}, - 'trivialprog2@exe': {'name': 'trivialprog2', 'sources': ['main.cpp'], 'extra_files': ['fileC.hpp']}, + 'trivialprog0@exe': {'name': 'trivialprog0', 'sources': ['main.cpp'], 'extra_files': ['main.hpp', 'fileA.hpp', 'fileB.hpp', 'fileC.hpp']}, + 'trivialprog1@exe': {'name': 'trivialprog1', 'sources': ['main.cpp'], 'extra_files': ['main.hpp', 'fileA.hpp']}, + 'trivialprog2@exe': {'name': 'trivialprog2', 'sources': ['main.cpp'], 'extra_files': ['fileB.hpp', 'fileC.hpp']}, 'trivialprog3@exe': {'name': 'trivialprog3', 'sources': ['main.cpp'], 'extra_files': ['main.hpp']}, 'trivialprog4@exe': {'name': 'trivialprog4', 'sources': ['main.cpp'], 'extra_files': ['main.hpp']}, - 'trivialprog5@exe': {'name': 'trivialprog5', 'sources': ['main.cpp'], 'extra_files': ['main.hpp', 'fileC.hpp']}, - 'trivialprog6@exe': {'name': 'trivialprog6', 'sources': ['main.cpp'], 'extra_files': ['main.hpp']}, - 'trivialprog7@exe': {'name': 'trivialprog7', 'sources': ['main.cpp'], 'extra_files': ['main.hpp']}, + 'trivialprog5@exe': {'name': 'trivialprog5', 'sources': ['main.cpp'], 'extra_files': ['fileB.hpp', 'fileC.hpp', 'main.hpp']}, + 'trivialprog6@exe': {'name': 'trivialprog6', 'sources': ['main.cpp'], 'extra_files': ['main.hpp', 'fileA.hpp']}, + 'trivialprog7@exe': {'name': 'trivialprog7', 'sources': ['main.cpp'], 'extra_files': ['main.hpp', 'fileA.hpp']}, 'trivialprog8@exe': {'name': 'trivialprog8', 'sources': ['main.cpp'], 'extra_files': []}, 'trivialprog9@exe': {'name': 'trivialprog9', 'sources': ['main.cpp'], 'extra_files': []}, 'trivialprog10@exe': {'name': 'trivialprog10', 'sources': ['main.cpp'], 'extra_files': []}, @@ -400,7 +426,26 @@ class RewriterTests(BasePlatformTests): # Check the written file out = self.rewrite(self.builddir, os.path.join(self.builddir, 'info.json')) - self.assertDictEqual(out, expected) + self.assertEqualIgnoreOrder(out, expected) + + def test_tricky_dataflow(self): + self.prime('7 tricky dataflow') + out = self.rewrite(self.builddir, os.path.join(self.builddir, 'addSrc.json')) + expected = { + 'target': { + 'tgt1@sha': {'name': 'tgt1', 'sources': ['foo.c', 'new.c'], 'extra_files': []}, + 'tgt2@exe': {'name': 'tgt2', 'sources': ['new.c', 'unknown'], 'extra_files': []}, + 'tgt3@exe': {'name': 'tgt3', 'sources': ['foo.c', 'new.c'], 'extra_files': []}, + 'tgt4@exe': {'name': 'tgt4', 'sources': ['unknown'], 'extra_files': []}, + 'tgt5@exe': {'name': 'tgt5', 'sources': ['unknown', 'new.c'], 'extra_files': []}, + 'tgt6@exe': {'name': 'tgt6', 'sources': ['unknown', 'new.c'], 'extra_files': []}, + } + } + self.assertEqualIgnoreOrder(out, expected) + + # Check the written file + out = self.rewrite(self.builddir, os.path.join(self.builddir, 'info.json')) + self.assertEqualIgnoreOrder(out, expected) def test_raw_printer_is_idempotent(self): test_path = Path(self.unit_test_dir, '120 rewrite') @@ -437,7 +482,7 @@ class RewriterTests(BasePlatformTests): # Asserts that AstInterpreter.dataflow_dag is what it should be def test_dataflow_dag(self): test_path = Path(self.rewrite_test_dir, '1 basic') - interpreter = IntrospectionInterpreter(test_path, '', 'ninja', visitors = [AstIDGenerator()]) + interpreter = IntrospectionInterpreter(test_path, '', 'ninja') interpreter.analyze() def sortkey(node): |