From 3a33a8ef49227c2fce1c0c7143e5529b4208d04e Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Fri, 9 Jun 2017 13:00:20 +0530 Subject: unit tests: Add class to generate failing tests It is not feasible to test all failure modes by creating projects in `test cases/failing` that would be an explosion of files, and that mechanism is too coarse anyway. We have no way to ensure that the expected error is being raised. See FailureTests.test_dependency for an example. --- run_project_tests.py | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) (limited to 'run_project_tests.py') diff --git a/run_project_tests.py b/run_project_tests.py index 822286b08..5a88fa47e 100755 --- a/run_project_tests.py +++ b/run_project_tests.py @@ -34,7 +34,7 @@ import time import multiprocessing import concurrent.futures as conc import re -from run_unittests import get_fake_options +from run_unittests import get_fake_options, run_configure_inprocess from run_tests import get_backend_commands, get_backend_args_for_dir, Backend from run_tests import ensure_backend_detects_changes @@ -249,18 +249,6 @@ def log_text_file(logfile, testdir, stdo, stde): executor.shutdown() raise StopException() -def run_configure_inprocess(commandlist): - old_stdout = sys.stdout - sys.stdout = mystdout = StringIO() - old_stderr = sys.stderr - sys.stderr = mystderr = StringIO() - try: - returncode = mesonmain.run(commandlist[0], commandlist[1:]) - finally: - sys.stdout = old_stdout - sys.stderr = old_stderr - return returncode, mystdout.getvalue(), mystderr.getvalue() - def run_test_inprocess(testdir): old_stdout = sys.stdout sys.stdout = mystdout = StringIO() -- cgit v1.2.3 From c1e9c757ebbe7a26b126ee8cb2106f6f8dffa322 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Fri, 9 Jun 2017 23:23:38 +0530 Subject: tests: Increase dependencies coverage a bit more --- mesonbuild/dependencies/ui.py | 7 ++-- run_project_tests.py | 2 +- run_unittests.py | 66 +++++++++++++++++++++++++++++++--- test cases/frameworks/4 qt/meson.build | 6 ++++ 4 files changed, 71 insertions(+), 10 deletions(-) (limited to 'run_project_tests.py') diff --git a/mesonbuild/dependencies/ui.py b/mesonbuild/dependencies/ui.py index 7a66a2d7c..8537a7ed9 100644 --- a/mesonbuild/dependencies/ui.py +++ b/mesonbuild/dependencies/ui.py @@ -427,7 +427,6 @@ class WxDependency(ExternalDependency): if WxDependency.wx_found is None: self.check_wxconfig() if not WxDependency.wx_found: - # FIXME: this message could be printed after Dependncy found mlog.log("Neither wx-config-3.0 nor wx-config found; can't detect dependency") return @@ -466,11 +465,11 @@ class WxDependency(ExternalDependency): if modules not in kwargs: return [] candidates = kwargs[modules] - if isinstance(candidates, str): - return [candidates] + if not isinstance(candidates, list): + candidates = [candidates] for c in candidates: if not isinstance(c, str): - raise DependencyException('wxwidgets module argument is not a string.') + raise DependencyException('wxwidgets module argument is not a string') return candidates def check_wxconfig(self): diff --git a/run_project_tests.py b/run_project_tests.py index 5a88fa47e..71d6c0cf8 100755 --- a/run_project_tests.py +++ b/run_project_tests.py @@ -434,7 +434,7 @@ def detect_tests_to_run(): ('objective c', 'objc', backend not in (Backend.ninja, Backend.xcode) or mesonlib.is_windows() or not have_objc_compiler()), ('fortran', 'fortran', backend is not Backend.ninja or not shutil.which('gfortran')), ('swift', 'swift', backend not in (Backend.ninja, Backend.xcode) or not shutil.which('swiftc')), - ('python3', 'python3', backend is not Backend.ninja or not shutil.which('python3')), + ('python3', 'python3', backend is not Backend.ninja), ] return [(name, gather_tests('test cases/' + subdir), skip) for name, subdir, skip in all_tests] diff --git a/run_unittests.py b/run_unittests.py index 96aefc589..bcb55a230 100755 --- a/run_unittests.py +++ b/run_unittests.py @@ -468,7 +468,7 @@ class BasePlatformTests(unittest.TestCase): self.privatedir = os.path.join(self.builddir, 'meson-private') if inprocess: try: - run_configure_inprocess(self.meson_args + args + extra_args) + out = run_configure_inprocess(self.meson_args + args + extra_args)[1] except: self._print_meson_log() raise @@ -479,12 +479,13 @@ class BasePlatformTests(unittest.TestCase): mesonbuild.mlog.log_file = None else: try: - self._run(self.meson_command + args + extra_args) + out = self._run(self.meson_command + args + extra_args) except unittest.SkipTest: raise unittest.SkipTest('Project requested skipping: ' + srcdir) except: self._print_meson_log() raise + return out def build(self, target=None, extra_args=None): if extra_args is None: @@ -1230,13 +1231,17 @@ class FailureTests(BasePlatformTests): super().tearDown() shutil.rmtree(self.srcdir) - def assertMesonRaises(self, contents, match, extra_args=None): + def assertMesonRaises(self, contents, match, extra_args=None, langs=None): ''' - Assert that running meson configure on the specified contents raises - the specified error message. + Assert that running meson configure on the specified @contents raises + a error message matching regex @match. ''' + if langs is None: + langs = [] with open(self.mbuild, 'w') as f: f.write("project('failure test', 'c', 'cpp')\n") + for lang in langs: + f.write("add_languages('{}', required : false)\n".format(lang)) f.write(contents) # Force tracebacks so we can detect them properly os.environ['MESON_FORCE_BACKTRACE'] = '1' @@ -1244,6 +1249,22 @@ class FailureTests(BasePlatformTests): # Must run in-process or we'll get a generic CalledProcessError self.init(self.srcdir, extra_args=extra_args, inprocess=True) + def assertMesonOutputs(self, contents, match, extra_args=None, langs=None): + ''' + Assert that running meson configure on the specified @contents outputs + something that matches regex @match. + ''' + if langs is None: + langs = [] + with open(self.mbuild, 'w') as f: + f.write("project('output test', 'c', 'cpp')\n") + for lang in langs: + f.write("add_languages('{}', required : false)\n".format(lang)) + f.write(contents) + # Run in-process for speed and consistency with assertMesonRaises + out = self.init(self.srcdir, extra_args=extra_args, inprocess=True) + self.assertRegex(out, match) + def test_dependency(self): if not shutil.which('pkg-config'): raise unittest.SkipTest('pkg-config not found') @@ -1256,6 +1277,41 @@ class FailureTests(BasePlatformTests): for contents, match in a: self.assertMesonRaises(contents, match) + def test_apple_frameworks_dependency(self): + if not is_osx(): + raise unittest.SkipTest('only run on macOS') + self.assertMesonRaises("dependency('appleframeworks')", + "requires at least one module") + + def test_sdl2_notfound_dependency(self): + # Want to test failure, so skip if available + if shutil.which('sdl2-config'): + raise unittest.SkipTest('sdl2-config found') + self.assertMesonRaises("dependency('sdl2', method : 'sdlconfig')", self.dnf) + self.assertMesonRaises("dependency('sdl2', method : 'pkg-config')", self.dnf) + + def test_gnustep_notfound_dependency(self): + # Want to test failure, so skip if available + if shutil.which('gnustep-config'): + raise unittest.SkipTest('gnustep-config found') + self.assertMesonRaises("dependency('gnustep')", + "(requires a Objc compiler|{})".format(self.dnf), + langs = ['objc']) + + def test_wx_notfound_dependency(self): + # Want to test failure, so skip if available + if shutil.which('wx-config-3.0') or shutil.which('wx-config'): + raise unittest.SkipTest('wx-config or wx-config-3.0 found') + self.assertMesonRaises("dependency('wxwidgets')", self.dnf) + self.assertMesonOutputs("dependency('wxwidgets', required : false)", + "nor wx-config found") + + def test_wx_dependency(self): + if not shutil.which('wx-config-3.0') and not shutil.which('wx-config'): + raise unittest.SkipTest('Neither wx-config nor wx-config-3.0 found') + self.assertMesonRaises("dependency('wxwidgets', modules : 1)", + "module argument is not a string") + def test_llvm_dependency(self): self.assertMesonRaises("dependency('llvm', modules : 'fail')", "(required.*fail|{})".format(self.dnf)) diff --git a/test cases/frameworks/4 qt/meson.build b/test cases/frameworks/4 qt/meson.build index 468b9c93b..d9cab6fbe 100644 --- a/test cases/frameworks/4 qt/meson.build +++ b/test cases/frameworks/4 qt/meson.build @@ -8,16 +8,22 @@ foreach qt : ['qt4', 'qt5'] if qt == 'qt5' qt_modules += qt5_modules endif + # Test that invalid modules are indeed not found fakeqtdep = dependency(qt, modules : ['DefinitelyNotFound'], required : false, method : get_option('method')) if fakeqtdep.found() error('Invalid qt dep incorrectly found!') endif + # Test that partially-invalid modules are indeed not found fakeqtdep = dependency(qt, modules : ['Core', 'DefinitelyNotFound'], required : false, method : get_option('method')) if fakeqtdep.found() error('Invalid qt dep incorrectly found!') endif + + # Ensure that the "no-Core-module-specified" code branch is hit + nocoredep = dependency(qt, modules : ['Gui'], required : qt == 'qt5', method : get_option('method')) + # If qt4 modules are found, test that. qt5 is required. qtdep = dependency(qt, modules : qt_modules, required : qt == 'qt5', method : get_option('method')) if qtdep.found() -- cgit v1.2.3 From 9308a6d923ecdfabb40d88e1590226172ea95e43 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Sat, 10 Jun 2017 13:04:00 +0530 Subject: tests: Add Boost unit tests and project tests on Windows Boost tests are disabled on Windows for now because the detection is actually completely broken. Once that's fixed (after the release) we can enable it again. --- .appveyor.yml | 2 ++ mesonbuild/dependencies/misc.py | 7 ++++--- run_project_tests.py | 13 ++++++++++--- run_unittests.py | 13 +++++++++++++ 4 files changed, 29 insertions(+), 6 deletions(-) (limited to 'run_project_tests.py') diff --git a/.appveyor.yml b/.appveyor.yml index 1e20a372c..655144599 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -55,6 +55,8 @@ skip_commits: install: - cmd: set "ORIG_PATH=%PATH%" + # Boost 1.56.0: https://www.appveyor.com/docs/build-environment/#boost + #- cmd: set "BOOST_ROOT=C:\Libraries\boost" # Use the x86 python only when building for x86 for the cpython tests. # For all other archs (including, say, arm), use the x64 python. - ps: (new-object net.webclient).DownloadFile('https://www.dropbox.com/s/bbzvepq85hv47x1/ninja.exe?dl=1', 'C:\projects\meson\ninja.exe') diff --git a/mesonbuild/dependencies/misc.py b/mesonbuild/dependencies/misc.py index 1356ec847..c24acf06a 100644 --- a/mesonbuild/dependencies/misc.py +++ b/mesonbuild/dependencies/misc.py @@ -126,8 +126,8 @@ class BoostDependency(ExternalDependency): def get_requested(self, kwargs): candidates = kwargs.get('modules', []) - if isinstance(candidates, str): - return [candidates] + if not isinstance(candidates, list): + candidates = [candidates] for c in candidates: if not isinstance(c, str): raise DependencyException('Boost module argument is not a string.') @@ -136,7 +136,8 @@ class BoostDependency(ExternalDependency): def validate_requested(self): for m in self.requested_modules: if m not in self.src_modules: - raise DependencyException('Requested Boost module "%s" not found.' % m) + msg = 'Requested Boost module {!r} not found' + raise DependencyException(msg.format(m)) def detect_version(self): try: diff --git a/run_project_tests.py b/run_project_tests.py index 71d6c0cf8..66d7eb073 100755 --- a/run_project_tests.py +++ b/run_project_tests.py @@ -371,7 +371,7 @@ def _run_test(testdir, test_build_dir, install_dir, extra_args, compiler, backen return TestResult(validate_install(testdir, install_dir, compiler), BuildStep.validate, stdo, stde, mesonlog, gen_time, build_time, test_time) def gather_tests(testdir): - tests = [t.replace('\\', '/').split('/', 2)[2] for t in glob(os.path.join(testdir, '*'))] + tests = [t.replace('\\', '/').split('/', 2)[2] for t in glob(testdir + '/*')] testlist = [(int(t.split()[0]), t) for t in tests] testlist.sort() tests = [os.path.join(testdir, t[1]) for t in testlist] @@ -425,7 +425,6 @@ def detect_tests_to_run(): ('platform-windows', 'windows', not mesonlib.is_windows() and not mesonlib.is_cygwin()), ('platform-linux', 'linuxlike', mesonlib.is_osx() or mesonlib.is_windows()), - ('framework', 'frameworks', mesonlib.is_osx() or mesonlib.is_windows() or mesonlib.is_cygwin()), ('java', 'java', backend is not Backend.ninja or mesonlib.is_osx() or not have_java()), ('C#', 'csharp', backend is not Backend.ninja or not shutil.which('mcs')), ('vala', 'vala', backend is not Backend.ninja or not shutil.which('valac')), @@ -436,7 +435,15 @@ def detect_tests_to_run(): ('swift', 'swift', backend not in (Backend.ninja, Backend.xcode) or not shutil.which('swiftc')), ('python3', 'python3', backend is not Backend.ninja), ] - return [(name, gather_tests('test cases/' + subdir), skip) for name, subdir, skip in all_tests] + gathered_tests = [(name, gather_tests('test cases/' + subdir), skip) for name, subdir, skip in all_tests] + if mesonlib.is_windows(): + # TODO: Set BOOST_ROOT in .appveyor.yml + gathered_tests += [('framework', ['test cases/frameworks/1 boost'], 'BOOST_ROOT' not in os.environ)] + elif mesonlib.is_osx() or mesonlib.is_cygwin(): + gathered_tests += [('framework', gather_tests('test cases/frameworks'), True)] + else: + gathered_tests += [('framework', gather_tests('test cases/frameworks'), False)] + return gathered_tests def run_tests(all_tests, log_name_base, extra_args): global stop, executor, futures diff --git a/run_unittests.py b/run_unittests.py index bcb55a230..dbfd63864 100755 --- a/run_unittests.py +++ b/run_unittests.py @@ -1316,6 +1316,19 @@ class FailureTests(BasePlatformTests): self.assertMesonRaises("dependency('llvm', modules : 'fail')", "(required.*fail|{})".format(self.dnf)) + def test_boost_notfound_dependency(self): + # Can be run even if Boost is found or not + self.assertMesonRaises("dependency('boost', modules : 1)", + "module.*not a string") + self.assertMesonRaises("dependency('boost', modules : 'fail')", + "(fail.*not found|{})".format(self.dnf)) + + def test_boost_BOOST_ROOT_dependency(self): + # Test BOOST_ROOT; can be run even if Boost is found or not + os.environ['BOOST_ROOT'] = 'relative/path' + self.assertMesonRaises("dependency('boost')", + "(BOOST_ROOT.*absolute|{})".format(self.dnf)) + class WindowsTests(BasePlatformTests): ''' -- cgit v1.2.3