From e3bc2e5c68c7590eaf5962fc87cfa587285900ef Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Fri, 1 Apr 2016 20:14:31 +0530 Subject: ninja: Set PATH for CustomTargets with built EXEs on Windows When a CustomTarget is run with a command that is an executable built by the project which also has a DLL built in the same project as a dependency, the EXE can't run on Windows because the DLL can't be found. On UNIX-like systems, we set the RPATH using the linker so these dependencies can be found, but on Windows the only way is to set the PATH environment variable. The same problem exists for tests, so we reuse that infrastructure by creating a new meson_exe.py script that can be used as a wrapper to run CustomTarget commands on Windows. This can later also be extended to add support for setting an environment while calling the command needed to generate a CustomTarget: https://github.com/mesonbuild/meson/issues/266 --- mesonbuild/backend/backends.py | 41 +++++++++++++++++++++ mesonbuild/backend/ninjabackend.py | 17 ++++++++- mesonbuild/mesonmain.py | 5 ++- mesonbuild/scripts/meson_exe.py | 74 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 135 insertions(+), 2 deletions(-) create mode 100644 mesonbuild/scripts/meson_exe.py diff --git a/mesonbuild/backend/backends.py b/mesonbuild/backend/backends.py index ceb466b62..08e3d2df2 100644 --- a/mesonbuild/backend/backends.py +++ b/mesonbuild/backend/backends.py @@ -34,6 +34,18 @@ class InstallData(): self.install_scripts = [] self.install_subdirs = [] +class ExecutableSerialisation(): + def __init__(self, name, fname, cmd_args, env, is_cross, exe_wrapper, + workdir, extra_paths): + self.name = name + self.fname = fname + self.cmd_args = cmd_args + self.env = env + self.is_cross = is_cross + self.exe_runner = exe_wrapper + self.workdir = workdir + self.extra_paths = extra_paths + class TestSerialisation: def __init__(self, name, suite, fname, is_cross, exe_wrapper, is_parallel, cmd_args, env, should_fail, valgrind_args, timeout, workdir, extra_paths): @@ -154,6 +166,35 @@ class Backend(): raise MesonException('Unknown data type in object list.') return obj_list + def serialise_executable(self, exe, cmd_args, workdir, env={}): + import uuid + # Can't just use exe.name here; it will likely be run more than once + scratch_file = 'meson_exe_{0}_{1}.dat'.format(exe.name, + str(uuid.uuid4())[:8]) + exe_data = os.path.join(self.environment.get_scratch_dir(), scratch_file) + with open(exe_data, 'wb') as f: + if isinstance(exe, dependencies.ExternalProgram): + exe_fullpath = exe.fullpath + else: + exe_fullpath = [os.path.join(self.environment.get_build_dir(), + self.get_target_filename(exe))] + is_cross = self.environment.is_cross_build() and \ + self.environment.cross_info.need_cross_compiler() and \ + self.environment.cross_info.need_exe_wrapper() + if is_cross: + exe_wrapper = self.environment.cross_info.config['binaries'].get('exe_wrapper', None) + else: + exe_wrapper = None + if mesonlib.is_windows(): + extra_paths = self.determine_windows_extra_paths(exe) + else: + extra_paths = [] + es = ExecutableSerialisation(exe.name, exe_fullpath, cmd_args, env, + is_cross, exe_wrapper, workdir, + extra_paths) + pickle.dump(es, f) + return exe_data + def serialise_tests(self): test_data = os.path.join(self.environment.get_scratch_dir(), 'meson_test_setup.dat') datafile = open(test_data, 'wb') diff --git a/mesonbuild/backend/ninjabackend.py b/mesonbuild/backend/ninjabackend.py index 201b2d14e..99643227a 100644 --- a/mesonbuild/backend/ninjabackend.py +++ b/mesonbuild/backend/ninjabackend.py @@ -341,6 +341,7 @@ int dummy; def generate_custom_target(self, target, outfile): (srcs, ofilenames, cmd) = self.eval_custom_target_command(target) deps = [] + desc = 'Generating {0} with a {1} command.' for i in target.get_dependencies(): # FIXME, should not grab element at zero but rather expand all. if isinstance(i, list): @@ -364,9 +365,23 @@ int dummy; tmp = [tmp] for fname in tmp: elem.add_dep(os.path.join(self.get_target_dir(d), fname)) + # Windows doesn't have -rpath, so for EXEs that need DLLs built within + # the project, we need to set PATH so the DLLs are found. We use + # a serialized executable wrapper for that and check if the + # CustomTarget command needs extra paths first. + if mesonlib.is_windows() and \ + self.determine_windows_extra_paths(target.command[0]): + exe_data = self.serialise_executable(target.command[0], cmd[1:], + # All targets are built from the build dir + self.environment.get_build_dir()) + cmd = [sys.executable, self.environment.get_build_command(), + '--internal', 'exe', exe_data] + cmd_type = 'meson_exe.py custom' + else: + cmd_type = 'custom' elem.add_item('COMMAND', cmd) - elem.add_item('description', 'Generating %s with a custom command.' % target.name) + elem.add_item('description', desc.format(target.name, cmd_type)) elem.write(outfile) self.processed_targets[target.name + target.type_suffix()] = True diff --git a/mesonbuild/mesonmain.py b/mesonbuild/mesonmain.py index 3b05afbd4..052c17838 100644 --- a/mesonbuild/mesonmain.py +++ b/mesonbuild/mesonmain.py @@ -163,7 +163,10 @@ itself as required.''' def run_script_command(args): cmdname = args[0] cmdargs = args[1:] - if cmdname == 'test': + if cmdname == 'exe': + import mesonbuild.scripts.meson_exe as abc + cmdfunc = abc.run + elif cmdname == 'test': import mesonbuild.scripts.meson_test as abc cmdfunc = abc.run elif cmdname == 'benchmark': diff --git a/mesonbuild/scripts/meson_exe.py b/mesonbuild/scripts/meson_exe.py new file mode 100644 index 000000000..2b91de87e --- /dev/null +++ b/mesonbuild/scripts/meson_exe.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 + +# Copyright 2013-2016 The Meson development team + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +import argparse +import pickle +import platform +import subprocess + +import mesonbuild + +options = None + +parser = argparse.ArgumentParser() +parser.add_argument('args', nargs='+') + +def is_windows(): + platname = platform.system().lower() + return platname == 'windows' or 'mingw' in platname + +def run_with_mono(fname): + if fname.endswith('.exe') and not is_windows(): + return True + return False + +def run_exe(exe): + if exe.fname[0].endswith('.jar'): + cmd = ['java', '-jar'] + exe.fname + elif not exe.is_cross and run_with_mono(exe.fname[0]): + cmd = ['mono'] + exe.fname + else: + if exe.is_cross: + if exe.exe_runner is None: + raise Exception('BUG: Trying to run cross-compiled exes with no wrapper') + else: + cmd = [exe.exe_runner] + exe.fname + else: + cmd = exe.fname + child_env = os.environ.copy() + child_env.update(exe.env) + if len(exe.extra_paths) > 0: + child_env['PATH'] = child_env['PATH'] + ';'.join([''] + exe.extra_paths) + p = subprocess.Popen(cmd + exe.cmd_args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=child_env, + cwd=exe.workdir) + +def run(args): + global options + options = parser.parse_args(args) + if len(options.args) != 1: + print('Test runner for Meson. Do not run on your own, mmm\'kay?') + print(sys.argv[0] + ' [data file]') + exe_data_file = options.args[0] + exe = pickle.load(open(exe_data_file, 'rb')) + run_exe(exe) + +if __name__ == '__main__': + sys.exit(run(sys.argv[1:])) -- cgit v1.2.3 From 86aa51232ee73343d6f0aa136e925aa196be55f3 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Fri, 15 Apr 2016 03:08:06 +0530 Subject: backends: Don't duplicate test setup data file name Instead, return the values of the test and benchmark setup data files so that the ninja/osx/vs backends can use those filenames instead of hard-coding them. --- mesonbuild/backend/backends.py | 1 + mesonbuild/backend/ninjabackend.py | 4 +--- mesonbuild/backend/vs2010backend.py | 5 +---- mesonbuild/backend/xcodebackend.py | 7 +++---- 4 files changed, 6 insertions(+), 11 deletions(-) diff --git a/mesonbuild/backend/backends.py b/mesonbuild/backend/backends.py index 08e3d2df2..12529b3e4 100644 --- a/mesonbuild/backend/backends.py +++ b/mesonbuild/backend/backends.py @@ -204,6 +204,7 @@ class Backend(): datafile = open(benchmark_data, 'wb') self.write_benchmark_file(datafile) datafile.close() + return (test_data, benchmark_data) def has_source_suffix(self, target, suffix): for s in target.get_sources(): diff --git a/mesonbuild/backend/ninjabackend.py b/mesonbuild/backend/ninjabackend.py index 99643227a..f39c02bec 100644 --- a/mesonbuild/backend/ninjabackend.py +++ b/mesonbuild/backend/ninjabackend.py @@ -597,10 +597,9 @@ int dummy; elem.write(outfile) def generate_tests(self, outfile): - self.serialise_tests() + (test_data, benchmark_data) = self.serialise_tests() valgrind = environment.find_valgrind() script_root = self.environment.get_script_dir() - test_data = os.path.join(self.environment.get_scratch_dir(), 'meson_test_setup.dat') cmd = [ sys.executable, self.environment.get_build_command(), '--internal', 'test' ] if not self.environment.coredata.get_builtin_option('stdsplit'): cmd += ['--no-stdsplit'] @@ -623,7 +622,6 @@ int dummy; # And then benchmarks. benchmark_script = os.path.join(script_root, 'meson_benchmark.py') - benchmark_data = os.path.join(self.environment.get_scratch_dir(), 'meson_benchmark_setup.dat') cmd = [sys.executable, self.environment.get_build_command(), '--internal', 'benchmark', benchmark_data] elem = NinjaBuildElement(self.all_outputs, 'benchmark', 'CUSTOM_COMMAND', ['all', 'PHONY']) elem.add_item('COMMAND', cmd) diff --git a/mesonbuild/backend/vs2010backend.py b/mesonbuild/backend/vs2010backend.py index 725b8ed72..5199ea5c3 100644 --- a/mesonbuild/backend/vs2010backend.py +++ b/mesonbuild/backend/vs2010backend.py @@ -773,7 +773,6 @@ if %%errorlevel%% neq 0 goto :VCEnd''' ET.SubElement(midl, 'ProxyFileName').text = '%(Filename)_p.c' postbuild = ET.SubElement(action, 'PostBuildEvent') ET.SubElement(postbuild, 'Message') - test_data = os.path.join(self.environment.get_scratch_dir(), 'meson_test_setup.dat') test_command = [sys.executable, self.environment.get_build_command(), '--internal', @@ -787,14 +786,12 @@ endlocal & call :cmErrorLevel %%errorlevel%% & goto :cmDone exit /b %%1 :cmDone if %%errorlevel%% neq 0 goto :VCEnd''' + test_data = self.serialise_tests()[0] ET.SubElement(postbuild, 'Command').text =\ cmd_templ % ('" "'.join(test_command), test_data) ET.SubElement(root, 'Import', Project='$(VCTargetsPath)\Microsoft.Cpp.targets') tree = ET.ElementTree(root) tree.write(ofname, encoding='utf-8', xml_declaration=True) - datafile = open(test_data, 'wb') - self.serialise_tests() - datafile.close() # ElementTree can not do prettyprinting so do it manually #doc = xml.dom.minidom.parse(ofname) #open(ofname, 'w').write(doc.toprettyxml()) diff --git a/mesonbuild/backend/xcodebackend.py b/mesonbuild/backend/xcodebackend.py index 6bda826a8..0ce90cedc 100644 --- a/mesonbuild/backend/xcodebackend.py +++ b/mesonbuild/backend/xcodebackend.py @@ -64,7 +64,7 @@ class XCodeBackend(backends.Backend): def generate(self, interp): self.interpreter = interp - self.serialise_tests() + test_data = self.serialise_tests()[0] self.generate_filemap() self.generate_buildmap() self.generate_buildstylemap() @@ -92,7 +92,7 @@ class XCodeBackend(backends.Backend): self.generate_pbx_group() self.generate_pbx_native_target() self.generate_pbx_project() - self.generate_pbx_shell_build_phase() + self.generate_pbx_shell_build_phase(test_data) self.generate_pbx_sources_build_phase() self.generate_pbx_target_dependency() self.generate_xc_build_configuration() @@ -480,7 +480,7 @@ class XCodeBackend(backends.Backend): self.write_line('};') self.ofile.write('/* End PBXProject section */\n') - def generate_pbx_shell_build_phase(self): + def generate_pbx_shell_build_phase(self, test_data): self.ofile.write('\n/* Begin PBXShellScriptBuildPhase section */\n') self.write_line('%s = {' % self.test_command_id) self.indent_level += 1 @@ -496,7 +496,6 @@ class XCodeBackend(backends.Backend): self.write_line('shellPath = /bin/sh;') script_root = self.environment.get_script_dir() test_script = os.path.join(script_root, 'meson_test.py') - test_data = os.path.join(self.environment.get_scratch_dir(), 'meson_test_setup.dat') cmd = [sys.executable, test_script, test_data, '--wd', self.environment.get_build_dir()] cmdstr = ' '.join(["'%s'" % i for i in cmd]) self.write_line('shellScript = "%s";' % cmdstr) -- cgit v1.2.3 From ed774a68c647eb00b8e531a8370161fb6696230b Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Mon, 18 Apr 2016 22:50:37 +0530 Subject: scripts/meson_exe: prepend extra_paths to PATH instead of appending This way locally-built DLLs and EXEs are preferred over system-wide ones --- mesonbuild/scripts/meson_exe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mesonbuild/scripts/meson_exe.py b/mesonbuild/scripts/meson_exe.py index 2b91de87e..f075fa082 100644 --- a/mesonbuild/scripts/meson_exe.py +++ b/mesonbuild/scripts/meson_exe.py @@ -53,7 +53,7 @@ def run_exe(exe): child_env = os.environ.copy() child_env.update(exe.env) if len(exe.extra_paths) > 0: - child_env['PATH'] = child_env['PATH'] + ';'.join([''] + exe.extra_paths) + child_env['PATH'] = ';'.join(exe.extra_paths + ['']) + child_env['PATH'] p = subprocess.Popen(cmd + exe.cmd_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, -- cgit v1.2.3