rewriter: Replace assignments with cur_assignments

Replace the variable tracking of `AstInterpreter.assignments`
with a slightly better variable tracking called
`AstInterpreter.cur_assignments`.
We now have a class `UnknownValue` for more explicit handling
of situations that are too complex/impossible.

5 files changed, 147 insertions, 29 deletions

diff --git a/mesonbuild/ast/interpreter.py b/mesonbuild/ast/interpreter.py
index f3ed7eb55..8dc5eb31f 100644
--- a/mesonbuild/ast/interpreter.py
+++ b/mesonbuild/ast/interpreter.py
@@ -8,6 +8,7 @@ from __future__ import annotations
 import os
 import sys
 import typing as T
+from collections import defaultdict
 from dataclasses import dataclass
 
 from .. import mparser, mesonlib
@@ -21,6 +22,7 @@ from ..interpreterbase import (
     ContinueRequest,
     Disabler,
     default_resolve_key,
+    UnknownValue,
 )
 
 from ..interpreter import (
@@ -44,6 +46,9 @@ from ..mparser import (
     NotNode,
     PlusAssignmentNode,
     TernaryNode,
+    SymbolNode,
+    Token,
+    FunctionNode,
 )
 
 if T.TYPE_CHECKING:
@@ -59,12 +64,14 @@ if T.TYPE_CHECKING:
         OrNode,
         TestCaseClauseNode,
         UMinusNode,
-        FunctionNode,
     )
 
 _T = T.TypeVar('_T')
 _V = T.TypeVar('_V')
 
+def _symbol(val: str) -> SymbolNode:
+    return SymbolNode(Token('', '', 0, 0, 0, (0, 0), val))
+
 # `IntrospectionDependency` is to the `IntrospectionInterpreter` what `Dependency` is to the normal `Interpreter`.
 @dataclass
 class IntrospectionDependency(MesonInterpreterObject):
@@ -97,7 +104,7 @@ class AstInterpreter(InterpreterBase):
         super().__init__(source_root, subdir, subproject, subproject_dir, env)
         self.visitors = visitors if visitors is not None else []
         self.nesting: T.List[int] = []
-        self.assignments: T.Dict[str, BaseNode] = {}
+        self.cur_assignments: T.DefaultDict[str, T.List[T.Tuple[T.List[int], T.Union[BaseNode, UnknownValue]]]] = defaultdict(list)
         self.assign_vals: T.Dict[str, T.Any] = {}
         self.reverse_assignment: T.Dict[str, BaseNode] = {}
         self.funcs.update({'project': self.func_do_nothing,
@@ -229,14 +236,6 @@ class AstInterpreter(InterpreterBase):
         self.argument_depth -= 1
         return {}
 
-    def evaluate_plusassign(self, node: PlusAssignmentNode) -> None:
-        assert isinstance(node, PlusAssignmentNode)
-        # Cheat by doing a reassignment
-        self.assignments[node.var_name.value] = node.value  # Save a reference to the value node
-        if node.value.ast_id:
-            self.reverse_assignment[node.value.ast_id] = node
-        self.assign_vals[node.var_name.value] = self.evaluate_statement(node.value)
-
     def evaluate_indexing(self, node: IndexNode) -> int:
         return 0
 
@@ -275,13 +274,83 @@ class AstInterpreter(InterpreterBase):
         self.evaluate_statement(cur.value)
         return False
 
+    def find_potential_writes(self, node: BaseNode) -> T.Set[str]:
+        if isinstance(node, mparser.ForeachClauseNode):
+            return {el.value for el in node.varnames} | self.find_potential_writes(node.block)
+        elif isinstance(node, mparser.CodeBlockNode):
+            ret = set()
+            for line in node.lines:
+                ret.update(self.find_potential_writes(line))
+            return ret
+        elif isinstance(node, (AssignmentNode, PlusAssignmentNode)):
+            return set([node.var_name.value]) | self.find_potential_writes(node.value)
+        elif isinstance(node, IdNode):
+            return set()
+        elif isinstance(node, ArrayNode):
+            ret = set()
+            for arg in node.args.arguments:
+                ret.update(self.find_potential_writes(arg))
+            return ret
+        elif isinstance(node, mparser.DictNode):
+            ret = set()
+            for k, v in node.args.kwargs.items():
+                ret.update(self.find_potential_writes(k))
+                ret.update(self.find_potential_writes(v))
+            return ret
+        elif isinstance(node, FunctionNode):
+            ret = set()
+            for arg in node.args.arguments:
+                ret.update(self.find_potential_writes(arg))
+            for arg in node.args.kwargs.values():
+                ret.update(self.find_potential_writes(arg))
+            return ret
+        elif isinstance(node, MethodNode):
+            ret = self.find_potential_writes(node.source_object)
+            for arg in node.args.arguments:
+                ret.update(self.find_potential_writes(arg))
+            for arg in node.args.kwargs.values():
+                ret.update(self.find_potential_writes(arg))
+            return ret
+        elif isinstance(node, ArithmeticNode):
+            return self.find_potential_writes(node.left) | self.find_potential_writes(node.right)
+        elif isinstance(node, (mparser.NumberNode, mparser.StringNode, mparser.BreakNode, mparser.BooleanNode, mparser.ContinueNode)):
+            return set()
+        elif isinstance(node, mparser.IfClauseNode):
+            if isinstance(node.elseblock, EmptyNode):
+                ret = set()
+            else:
+                ret = self.find_potential_writes(node.elseblock.block)
+            for i in node.ifs:
+                ret.update(self.find_potential_writes(i))
+            return ret
+        elif isinstance(node, mparser.IndexNode):
+            return self.find_potential_writes(node.iobject) | self.find_potential_writes(node.index)
+        elif isinstance(node, mparser.IfNode):
+            return self.find_potential_writes(node.condition) | self.find_potential_writes(node.block)
+        elif isinstance(node, (mparser.ComparisonNode, mparser.OrNode, mparser.AndNode)):
+            return self.find_potential_writes(node.left) | self.find_potential_writes(node.right)
+        elif isinstance(node, mparser.NotNode):
+            return self.find_potential_writes(node.value)
+        elif isinstance(node, mparser.TernaryNode):
+            return self.find_potential_writes(node.condition) | self.find_potential_writes(node.trueblock) | self.find_potential_writes(node.falseblock)
+        elif isinstance(node, mparser.UMinusNode):
+            return self.find_potential_writes(node.value)
+        elif isinstance(node, mparser.ParenthesizedNode):
+            return self.find_potential_writes(node.inner)
+        raise mesonlib.MesonBugException('Unhandled node type')
+
     def evaluate_foreach(self, node: ForeachClauseNode) -> None:
+        asses = self.find_potential_writes(node)
+        for ass in asses:
+            self.cur_assignments[ass].append((self.nesting.copy(), UnknownValue()))
         try:
             self.evaluate_codeblock(node.block)
         except ContinueRequest:
             pass
         except BreakRequest:
             pass
+        for ass in asses:
+            self.cur_assignments[ass].append((self.nesting.copy(), UnknownValue())) # In case the foreach loops 0 times.
 
     def evaluate_if(self, node: IfClauseNode) -> None:
         self.nesting.append(0)
@@ -291,26 +360,66 @@ class AstInterpreter(InterpreterBase):
         if not isinstance(node.elseblock, EmptyNode):
             self.evaluate_codeblock(node.elseblock.block)
         self.nesting.pop()
+        for var_name in self.cur_assignments:
+            potential_values = []
+            oldval = self.get_cur_value(var_name, allow_none=True)
+            if oldval is not None:
+                potential_values.append(oldval)
+            for nesting, value in self.cur_assignments[var_name]:
+                if len(nesting) > len(self.nesting):
+                    potential_values.append(value)
+            self.cur_assignments[var_name] = [(nesting, v) for (nesting, v) in self.cur_assignments[var_name] if len(nesting) <= len(self.nesting)]
+            if len(potential_values) > 1 or (len(potential_values) > 0 and oldval is None):
+                uv = UnknownValue()
+                self.cur_assignments[var_name].append((self.nesting.copy(), uv))
+
+    def get_cur_value(self, var_name: str, allow_none: bool = False) -> T.Union[BaseNode, UnknownValue, None]:
+        if var_name in {'meson', 'host_machine', 'build_machine', 'target_machine'}:
+            return UnknownValue()
+        ret = None
+        for nesting, value in reversed(self.cur_assignments[var_name]):
+            if len(self.nesting) >= len(nesting) and self.nesting[:len(nesting)] == nesting:
+                ret = value
+                break
+        if ret is None and allow_none:
+            return ret
+        assert ret is not None
+        return ret
 
     def get_variable(self, varname: str) -> int:
         return 0
 
     def assignment(self, node: AssignmentNode) -> None:
         assert isinstance(node, AssignmentNode)
-        self.assignments[node.var_name.value] = node.value # Save a reference to the value node
+        self.cur_assignments[node.var_name.value].append((self.nesting.copy(), node.value))
         if node.value.ast_id:
             self.reverse_assignment[node.value.ast_id] = node
         self.assign_vals[node.var_name.value] = self.evaluate_statement(node.value) # Evaluate the value just in case
 
+    def evaluate_plusassign(self, node: PlusAssignmentNode) -> None:
+        assert isinstance(node, PlusAssignmentNode)
+        self.evaluate_statement(node.value)
+        lhs = self.get_cur_value(node.var_name.value)
+        newval: T.Union[UnknownValue, ArithmeticNode]
+        if isinstance(lhs, UnknownValue):
+            newval = UnknownValue()
+        else:
+            newval = mparser.ArithmeticNode(operation='add', left=lhs, operator=_symbol('+'), right=node.value)
+        self.cur_assignments[node.var_name.value].append((self.nesting.copy(), newval))
+
+        if node.value.ast_id:
+            self.reverse_assignment[node.value.ast_id] = node
+        self.assign_vals[node.var_name.value] = self.evaluate_statement(node.value)
+
     def resolve_node(self, node: BaseNode, include_unknown_args: bool = False, id_loop_detect: T.Optional[T.List[str]] = None) -> T.Optional[T.Any]:
         def quick_resolve(n: BaseNode, loop_detect: T.Optional[T.List[str]] = None) -> T.Any:
             if loop_detect is None:
                 loop_detect = []
             if isinstance(n, IdNode):
                 assert isinstance(n.value, str)
-                if n.value in loop_detect or n.value not in self.assignments:
+                if n.value in loop_detect or n.value not in self.cur_assignments:
                     return []
-                return quick_resolve(self.assignments[n.value], loop_detect = loop_detect + [n.value])
+                return quick_resolve(self.get_cur_value(n.value), loop_detect = loop_detect + [n.value])
             elif isinstance(n, ElementaryNode):
                 return n.value
             else:
diff --git a/mesonbuild/ast/introspection.py b/mesonbuild/ast/introspection.py
index b86b4e1e0..b9299ab78 100644
--- a/mesonbuild/ast/introspection.py
+++ b/mesonbuild/ast/introspection.py
@@ -12,7 +12,7 @@ import typing as T
 from .. import compilers, environment, mesonlib, options
 from ..build import Executable, Jar, SharedLibrary, SharedModule, StaticLibrary
 from ..compilers import detect_compiler_for
-from ..interpreterbase import InvalidArguments, SubProject
+from ..interpreterbase import InvalidArguments, SubProject, UnknownValue
 from ..mesonlib import MachineChoice
 from ..options import OptionKey
 from ..mparser import BaseNode, ArithmeticNode, ArrayNode, ElementaryNode, IdNode, FunctionNode, StringNode
@@ -179,7 +179,7 @@ class IntrospectionInterpreter(AstInterpreter):
     def func_add_languages(self, node: BaseNode, args: T.List[TYPE_var], kwargs: T.Dict[str, TYPE_var]) -> None:
         kwargs = self.flatten_kwargs(kwargs)
         required = kwargs.get('required', True)
-        assert isinstance(required, (bool, options.UserFeatureOption)), 'for mypy'
+        assert isinstance(required, (bool, options.UserFeatureOption, UnknownValue)), 'for mypy'
         if isinstance(required, options.UserFeatureOption):
             required = required.is_enabled()
         if 'native' in kwargs:
@@ -189,7 +189,7 @@ class IntrospectionInterpreter(AstInterpreter):
             for for_machine in [MachineChoice.BUILD, MachineChoice.HOST]:
                 self._add_languages(args, required, for_machine)
 
-    def _add_languages(self, raw_langs: T.List[TYPE_var], required: bool, for_machine: MachineChoice) -> None:
+    def _add_languages(self, raw_langs: T.List[TYPE_var], required: T.Union[bool, UnknownValue], for_machine: MachineChoice) -> None:
         langs: T.List[str] = []
         for l in self.flatten_args(raw_langs):
             if isinstance(l, str):
@@ -204,7 +204,7 @@ class IntrospectionInterpreter(AstInterpreter):
                     comp = detect_compiler_for(self.environment, lang, for_machine, True, self.subproject)
                 except mesonlib.MesonException:
                     # do we even care about introspecting this language?
-                    if required:
+                    if isinstance(required, UnknownValue) or required:
                         raise
                     else:
                         continue
@@ -271,8 +271,8 @@ class IntrospectionInterpreter(AstInterpreter):
                     # Try to resolve the ID and append the node to the queue
                     assert isinstance(curr.value, str)
                     var_name = curr.value
-                    if var_name in self.assignments:
-                        tmp_node = self.assignments[var_name]
+                    if var_name in self.cur_assignments:
+                        tmp_node = self.get_cur_value(var_name)
                         if isinstance(tmp_node, (ArrayNode, IdNode, FunctionNode)):
                             inqueue += [tmp_node]
                 elif isinstance(curr, ArithmeticNode):
diff --git a/mesonbuild/interpreterbase/__init__.py b/mesonbuild/interpreterbase/__init__.py
index aa38e9490..473379307 100644
--- a/mesonbuild/interpreterbase/__init__.py
+++ b/mesonbuild/interpreterbase/__init__.py
@@ -59,6 +59,8 @@ __all__ = [
     'TYPE_HoldableTypes',
 
     'HoldableTypes',
+
+    'UnknownValue',
 ]
 
 from .baseobjects import (
@@ -81,6 +83,8 @@ from .baseobjects import (
     SubProject,
 
     HoldableTypes,
+
+    UnknownValue,
 )
 
 from .decorators import (
diff --git a/mesonbuild/interpreterbase/baseobjects.py b/mesonbuild/interpreterbase/baseobjects.py
index a5ccccedc..7cda5729f 100644
--- a/mesonbuild/interpreterbase/baseobjects.py
+++ b/mesonbuild/interpreterbase/baseobjects.py
@@ -121,6 +121,12 @@ class MesonInterpreterObject(InterpreterObject):
 class MutableInterpreterObject:
     ''' Dummy class to mark the object type as mutable '''
 
+class UnknownValue(MesonInterpreterObject):
+    '''This class is only used for the rewriter/static introspection tool and
+    indicates that a value cannot be determined statically, either because of
+    limitations in our code or because the value differs from machine to
+    machine.'''
+
 HoldableTypes = (HoldableObject, int, bool, str, list, dict)
 TYPE_HoldableTypes = T.Union[TYPE_var, HoldableObject]
 InterpreterObjectTypeVar = T.TypeVar('InterpreterObjectTypeVar', bound=TYPE_HoldableTypes)
diff --git a/mesonbuild/rewriter.py b/mesonbuild/rewriter.py
index fff3f9cfc..3360f6fc9 100644
--- a/mesonbuild/rewriter.py
+++ b/mesonbuild/rewriter.py
@@ -10,12 +10,12 @@
 from __future__ import annotations
 
 from .ast import IntrospectionInterpreter, BUILD_TARGET_FUNCTIONS, AstConditionLevel, AstIDGenerator, AstIndentationGenerator, AstPrinter
-from .ast.interpreter import IntrospectionBuildTarget, IntrospectionDependency
+from .ast.interpreter import IntrospectionBuildTarget, IntrospectionDependency, _symbol
 from .interpreterbase import TV_func
 from mesonbuild.mesonlib import MesonException, setup_vsenv
 from . import mlog, environment
 from functools import wraps
-from .mparser import Token, ArrayNode, ArgumentNode, AssignmentNode, BaseNode, StringNode, BooleanNode, ElementaryNode, IdNode, FunctionNode, SymbolNode
+from .mparser import Token, ArrayNode, ArgumentNode, AssignmentNode, BaseNode, StringNode, BooleanNode, ElementaryNode, IdNode, FunctionNode
 import json, os, re, sys
 import typing as T
 
@@ -97,9 +97,6 @@ class RequiredKeys:
 
         return T.cast('TV_func', wrapped)
 
-def _symbol(val: str) -> SymbolNode:
-    return SymbolNode(Token('', '', 0, 0, 0, (0, 0), val))
-
 class MTypeBase:
     node: BaseNode
 
@@ -434,8 +431,8 @@ class Rewriter:
 
         # Check the assignments
         tgt = None
-        if target in self.interpreter.assignments:
-            node = self.interpreter.assignments[target]
+        if target in self.interpreter.cur_assignments:
+            node = self.interpreter.get_cur_value(target)
             if isinstance(node, FunctionNode):
                 if node.func_name.value in {'executable', 'jar', 'library', 'shared_library', 'shared_module', 'static_library', 'both_libraries'}:
                     tgt = self.interpreter.assign_vals[target]
@@ -455,8 +452,8 @@ class Rewriter:
             return dep
 
         # Check the assignments
-        if dependency in self.interpreter.assignments:
-            node = self.interpreter.assignments[dependency]
+        if dependency in self.interpreter.cur_assignments:
+            node = self.interpreter.get_cur_value(dependency)
             if isinstance(node, FunctionNode):
                 if node.func_name.value == 'dependency':
                     name = self.interpreter.flatten_args(node.args)[0]
@@ -759,7 +756,9 @@ class Rewriter:
                         self.modified_nodes += [tgt_function]
                 target.extra_files = [node]
             if isinstance(node, IdNode):
-                node = self.interpreter.assignments[node.value]
+                cv = self.interpreter.get_cur_value(node.value)
+                assert isinstance(cv, BaseNode)
+                node = cv
                 target.extra_files = [node]
             if not isinstance(node, ArrayNode):
                 mlog.error('Target', mlog.bold(cmd['target']), 'extra_files argument must be a list', *self.on_error())