diff options
36 files changed, 64 insertions, 62 deletions
diff --git a/data/test.schema.json b/data/test.schema.json index 6f6a19381..705413363 100644 --- a/data/test.schema.json +++ b/data/test.schema.json @@ -87,13 +87,13 @@ "type": "string" } }, - "skip_on_jobname": { + "expect_skip_on_jobname": { "type": "array", "items": { "type": "string" } }, - "skip_on_os": { + "expect_skip_on_os": { "type": "array", "items": { "type": "string" @@ -167,13 +167,13 @@ "type": "string" } }, - "skip_on_jobname": { + "expect_skip_on_jobname": { "type": "array", "items": { "type": "string" } }, - "skip_on_os": { + "expect_skip_on_os": { "type": "array", "items": { "type": "string" diff --git a/docs/markdown/Contributing.md b/docs/markdown/Contributing.md index 68e943bff..731abc87d 100644 --- a/docs/markdown/Contributing.md +++ b/docs/markdown/Contributing.md @@ -334,9 +334,11 @@ Each value must contain the `val` key for the value of the option. `null` can be used for adding matrix entries without the current option. -The `skip_on_env`, `skip_on_jobname` and `skip_on_os` keys (as described below) -may be used in the value to skip that matrix entry, based on the current -environment. +The `skip_on_env` key (as described below) may be used in the value to skip that +matrix entry, based on the current environment. + +The `expect_skip_on_jobname` and `expect_skip_on_os` keys (as described below) +may be used to expect that the test will be skipped, based on the current environment. Similarly, the `compilers` key can be used to define a mapping of compilers to languages that are required for this value. @@ -412,23 +414,23 @@ The `skip_on_env` key can be used to specify a list of environment variables. If at least one environment variable in the `skip_on_env` list is present, the test is skipped. -#### skip_on_jobname +#### expect_skip_on_jobname -The `skip_on_jobname` key contains a list of strings. If the `MESON_CI_JOBNAME` +The `expect_skip_on_jobname` key contains a list of strings. If the `MESON_CI_JOBNAME` environment variable is set, and any of them are a sub-string of it, the test is expected to be skipped (that is, it is expected that the test will output `MESON_SKIP_TEST`, because the CI environment is not one in which it can run, for whatever reason). -The test is failed if it skips or runs unexpectedly. +The test is failed if it either skips unexpectedly or runs unexpectedly. -#### skip_on_os +#### expect_skip_on_os -The `skip_on_os` key can be used to specify a list of OS names (or their -negations, prefixed with a `!`). If at least one item in the `skip_on_os` list +The `expect_skip_on_os` key can be used to specify a list of OS names (or their +negations, prefixed with a `!`). If at least one item in the `expect_skip_on_os` list is matched, the test is expected to be skipped. -The test is failed if it skips or runs unexpectedly. +The test is failed if it either skips unexpectedly or runs unexpectedly. ### Skipping integration tests diff --git a/run_project_tests.py b/run_project_tests.py index a14741364..b159b3d6e 100755 --- a/run_project_tests.py +++ b/run_project_tests.py @@ -785,13 +785,13 @@ def _skip_keys(test_def: T.Dict) -> T.Tuple[bool, bool]: # Test is expected to skip if MESON_CI_JOBNAME contains any of the list of # substrings - if ('skip_on_jobname' in test_def) and (ci_jobname is not None): - skip_expected = any(s in ci_jobname for s in test_def['skip_on_jobname']) + if ('expect_skip_on_jobname' in test_def) and (ci_jobname is not None): + skip_expected = any(s in ci_jobname for s in test_def['expect_skip_on_jobname']) # Test is expected to skip if os matches - if 'skip_on_os' in test_def: + if 'expect_skip_on_os' in test_def: mesonenv = environment.Environment('', '', get_fake_options('/')) - for skip_os in test_def['skip_on_os']: + for skip_os in test_def['expect_skip_on_os']: if skip_os.startswith('!'): if mesonenv.machines.host.system != skip_os[1:]: skip_expected = True diff --git a/test cases/frameworks/1 boost/test.json b/test cases/frameworks/1 boost/test.json index 2c5b857e9..a09f7aa5d 100644 --- a/test cases/frameworks/1 boost/test.json +++ b/test cases/frameworks/1 boost/test.json @@ -18,5 +18,5 @@ { "static": "false", "b_vscrt": "mtd" } ] }, - "skip_on_jobname": ["azure", "msys2"] + "expect_skip_on_jobname": ["azure", "msys2"] } diff --git a/test cases/frameworks/10 gtk-doc/test.json b/test cases/frameworks/10 gtk-doc/test.json index 1085b55ee..796a60bdf 100644 --- a/test cases/frameworks/10 gtk-doc/test.json +++ b/test cases/frameworks/10 gtk-doc/test.json @@ -60,5 +60,5 @@ {"type": "file", "file": "usr/share/gtk-doc/html/foobar3/up.png"}, {"type": "file", "file": "usr/share/gtk-doc/html/foobar3/up-insensitive.png"} ], - "skip_on_jobname": ["azure", "msys2"] + "expect_skip_on_jobname": ["azure", "msys2"] } diff --git a/test cases/frameworks/11 gir subproject/test.json b/test cases/frameworks/11 gir subproject/test.json index aed0a1c71..7c4be111c 100644 --- a/test cases/frameworks/11 gir subproject/test.json +++ b/test cases/frameworks/11 gir subproject/test.json @@ -9,5 +9,5 @@ {"type": "expr", "file": "usr/lib/?libgirlib.so"}, {"type": "file", "platform": "cygwin", "file": "usr/lib/libgirsubproject.dll.a"} ], - "skip_on_jobname": ["azure", "cygwin", "macos", "msys2", "pypy"] + "expect_skip_on_jobname": ["azure", "cygwin", "macos", "msys2", "pypy"] } diff --git a/test cases/frameworks/12 multiple gir/test.json b/test cases/frameworks/12 multiple gir/test.json index e59c3e6cd..877c7c7eb 100644 --- a/test cases/frameworks/12 multiple gir/test.json +++ b/test cases/frameworks/12 multiple gir/test.json @@ -7,7 +7,7 @@ {"type": "expr", "file": "usr/lib/?libgirsubproject.so"}, {"type": "file", "platform": "cygwin", "file": "usr/lib/libgirsubproject.dll.a"} ], - "skip_on_jobname": ["azure", "macos", "msys2"], + "expect_skip_on_jobname": ["azure", "macos", "msys2"], "stdout": [ { "comment": "This will either match in the future-deprecated notice summary, or match the warning summary", diff --git a/test cases/frameworks/13 yelp/test.json b/test cases/frameworks/13 yelp/test.json index 22e34d20d..9410bdc58 100644 --- a/test cases/frameworks/13 yelp/test.json +++ b/test cases/frameworks/13 yelp/test.json @@ -19,5 +19,5 @@ {"type": "file", "file": "usr/share/help/de/meson-linguas/index3.page"}, {"type": "file", "file": "usr/share/help/de/meson-linguas/media/test.txt"} ], - "skip_on_jobname": ["azure", "cygwin", "macos", "msys2"] + "expect_skip_on_jobname": ["azure", "cygwin", "macos", "msys2"] } diff --git a/test cases/frameworks/14 doxygen/test.json b/test cases/frameworks/14 doxygen/test.json index c8c4fb0f8..47d4c16bb 100644 --- a/test cases/frameworks/14 doxygen/test.json +++ b/test cases/frameworks/14 doxygen/test.json @@ -2,5 +2,5 @@ "installed": [ {"type": "dir", "file": "usr/share/doc/spede/html"} ], - "skip_on_jobname": ["azure", "cygwin", "macos", "msys2"] + "expect_skip_on_jobname": ["azure", "cygwin", "macos", "msys2"] } diff --git a/test cases/frameworks/15 llvm/test.json b/test cases/frameworks/15 llvm/test.json index cf8aa3acc..f9bb64580 100644 --- a/test cases/frameworks/15 llvm/test.json +++ b/test cases/frameworks/15 llvm/test.json @@ -2,17 +2,17 @@ "matrix": { "options": { "method": [ - { "val": "config-tool", "skip_on_jobname": ["msys2-gcc"] }, - { "val": "cmake", "skip_on_jobname": ["msys2"] }, - { "val": "combination", "skip_on_jobname": ["msys2"] } + { "val": "config-tool", "expect_skip_on_jobname": ["msys2-gcc"] }, + { "val": "cmake", "expect_skip_on_jobname": ["msys2"] }, + { "val": "combination", "expect_skip_on_jobname": ["msys2"] } ], "link-static": [ - { "val": true, "skip_on_jobname": ["arch", "opensuse", "linux-gentoo-gcc"] }, + { "val": true, "expect_skip_on_jobname": ["arch", "opensuse", "linux-gentoo-gcc"] }, { "val": false } ] } }, - "skip_on_jobname": ["azure", "cygwin"], + "expect_skip_on_jobname": ["azure", "cygwin"], "tools": { "cmake": ">=3.11" } diff --git a/test cases/frameworks/16 sdl2/test.json b/test cases/frameworks/16 sdl2/test.json index ee4112ae3..f19ba609a 100644 --- a/test cases/frameworks/16 sdl2/test.json +++ b/test cases/frameworks/16 sdl2/test.json @@ -6,10 +6,10 @@ { "val": "pkg-config" }, { "val": "config-tool" }, { "val": "sdlconfig" }, - { "val": "extraframework", "skip_on_os": ["!darwin"], "skip_on_jobname": ["macos"] }, - { "val": "cmake", "skip_on_jobname": ["bionic"] } + { "val": "extraframework", "expect_skip_on_os": ["!darwin"], "expect_skip_on_jobname": ["macos"] }, + { "val": "cmake", "expect_skip_on_jobname": ["bionic"] } ] } }, - "skip_on_jobname": ["azure", "cygwin", "msys2"] + "expect_skip_on_jobname": ["azure", "cygwin", "msys2"] } diff --git a/test cases/frameworks/17 mpi/test.json b/test cases/frameworks/17 mpi/test.json index 115f6f6f5..cbd168612 100644 --- a/test cases/frameworks/17 mpi/test.json +++ b/test cases/frameworks/17 mpi/test.json @@ -5,7 +5,7 @@ { "val": "auto" }, { "val": "pkg-config" }, { "val": "config-tool", - "skip_on_jobname": ["fedora"] }, + "expect_skip_on_jobname": ["fedora"] }, { "val": "system", "compilers": { "c" :"msvc", "cpp": "msvc" } @@ -13,5 +13,5 @@ ] } }, - "skip_on_jobname": ["azure", "cygwin", "msys2", "opensuse"] + "expect_skip_on_jobname": ["azure", "cygwin", "msys2", "opensuse"] } diff --git a/test cases/frameworks/18 vulkan/test.json b/test cases/frameworks/18 vulkan/test.json index 6ace9de70..66afb97c3 100644 --- a/test cases/frameworks/18 vulkan/test.json +++ b/test cases/frameworks/18 vulkan/test.json @@ -1,3 +1,3 @@ { - "skip_on_jobname": ["azure", "cygwin", "macos", "msys2"] + "expect_skip_on_jobname": ["azure", "cygwin", "macos", "msys2"] } diff --git a/test cases/frameworks/19 pcap/test.json b/test cases/frameworks/19 pcap/test.json index 8ee026a60..23af7d931 100644 --- a/test cases/frameworks/19 pcap/test.json +++ b/test cases/frameworks/19 pcap/test.json @@ -1,3 +1,3 @@ { - "skip_on_jobname": ["azure", "cygwin", "msys2"] + "expect_skip_on_jobname": ["azure", "cygwin", "msys2"] } diff --git a/test cases/frameworks/2 gtest/test.json b/test cases/frameworks/2 gtest/test.json index 6ace9de70..66afb97c3 100644 --- a/test cases/frameworks/2 gtest/test.json +++ b/test cases/frameworks/2 gtest/test.json @@ -1,3 +1,3 @@ { - "skip_on_jobname": ["azure", "cygwin", "macos", "msys2"] + "expect_skip_on_jobname": ["azure", "cygwin", "macos", "msys2"] } diff --git a/test cases/frameworks/20 cups/test.json b/test cases/frameworks/20 cups/test.json index 8ee026a60..23af7d931 100644 --- a/test cases/frameworks/20 cups/test.json +++ b/test cases/frameworks/20 cups/test.json @@ -1,3 +1,3 @@ { - "skip_on_jobname": ["azure", "cygwin", "msys2"] + "expect_skip_on_jobname": ["azure", "cygwin", "msys2"] } diff --git a/test cases/frameworks/21 libwmf/test.json b/test cases/frameworks/21 libwmf/test.json index 6ace9de70..66afb97c3 100644 --- a/test cases/frameworks/21 libwmf/test.json +++ b/test cases/frameworks/21 libwmf/test.json @@ -1,3 +1,3 @@ { - "skip_on_jobname": ["azure", "cygwin", "macos", "msys2"] + "expect_skip_on_jobname": ["azure", "cygwin", "macos", "msys2"] } diff --git a/test cases/frameworks/22 gir link order/test.json b/test cases/frameworks/22 gir link order/test.json index a9d74fb8a..b15798127 100644 --- a/test cases/frameworks/22 gir link order/test.json +++ b/test cases/frameworks/22 gir link order/test.json @@ -1,3 +1,3 @@ { - "skip_on_jobname": ["azure", "macos", "msys2"] + "expect_skip_on_jobname": ["azure", "macos", "msys2"] } diff --git a/test cases/frameworks/23 hotdoc/test.json b/test cases/frameworks/23 hotdoc/test.json index e13971ee8..2a1f3a93c 100644 --- a/test cases/frameworks/23 hotdoc/test.json +++ b/test cases/frameworks/23 hotdoc/test.json @@ -5,5 +5,5 @@ "tools": { "hotdoc": ">=0.1.0" }, - "skip_on_jobname": ["msys2"] + "expect_skip_on_jobname": ["msys2"] } diff --git a/test cases/frameworks/24 libgcrypt/test.json b/test cases/frameworks/24 libgcrypt/test.json index 77b68c00c..de9b738f6 100644 --- a/test cases/frameworks/24 libgcrypt/test.json +++ b/test cases/frameworks/24 libgcrypt/test.json @@ -1,3 +1,3 @@ { - "skip_on_jobname": ["azure", "msys2"] + "expect_skip_on_jobname": ["azure", "msys2"] } diff --git a/test cases/frameworks/25 hdf5/test.json b/test cases/frameworks/25 hdf5/test.json index a5c816f66..590d4bc20 100644 --- a/test cases/frameworks/25 hdf5/test.json +++ b/test cases/frameworks/25 hdf5/test.json @@ -2,10 +2,10 @@ "matrix": { "options": { "method": [ - { "val": "pkg-config", "skip_on_jobname": ["linux-gentoo-gcc"] }, - { "val": "config-tool", "skip_on_jobname": ["macos"] } + { "val": "pkg-config", "expect_skip_on_jobname": ["linux-gentoo-gcc"] }, + { "val": "config-tool", "expect_skip_on_jobname": ["macos"] } ] } }, - "skip_on_jobname": ["azure", "cygwin", "fedora", "msys2", "opensuse"] + "expect_skip_on_jobname": ["azure", "cygwin", "fedora", "msys2", "opensuse"] } diff --git a/test cases/frameworks/26 netcdf/test.json b/test cases/frameworks/26 netcdf/test.json index 83c629167..3cf520750 100644 --- a/test cases/frameworks/26 netcdf/test.json +++ b/test cases/frameworks/26 netcdf/test.json @@ -1,3 +1,3 @@ { - "skip_on_jobname": ["azure", "bionic", "cygwin", "fedora", "macos", "msys2", "opensuse", "ubuntu"] + "expect_skip_on_jobname": ["azure", "bionic", "cygwin", "fedora", "macos", "msys2", "opensuse", "ubuntu"] } diff --git a/test cases/frameworks/27 gpgme/test.json b/test cases/frameworks/27 gpgme/test.json index 59eb0e24d..f64d7189e 100644 --- a/test cases/frameworks/27 gpgme/test.json +++ b/test cases/frameworks/27 gpgme/test.json @@ -1,3 +1,3 @@ { - "skip_on_jobname": ["azure", "cygwin", "macos", "msys2", "linux-arch", "ubuntu"] + "expect_skip_on_jobname": ["azure", "cygwin", "macos", "msys2", "linux-arch", "ubuntu"] } diff --git a/test cases/frameworks/28 gir link order 2/test.json b/test cases/frameworks/28 gir link order 2/test.json index a9d74fb8a..b15798127 100644 --- a/test cases/frameworks/28 gir link order 2/test.json +++ b/test cases/frameworks/28 gir link order 2/test.json @@ -1,3 +1,3 @@ { - "skip_on_jobname": ["azure", "macos", "msys2"] + "expect_skip_on_jobname": ["azure", "macos", "msys2"] } diff --git a/test cases/frameworks/29 blocks/test.json b/test cases/frameworks/29 blocks/test.json index 34a8c415d..3bc0974da 100644 --- a/test cases/frameworks/29 blocks/test.json +++ b/test cases/frameworks/29 blocks/test.json @@ -1,3 +1,3 @@ { - "skip_on_jobname": ["azure", "gcc", "msys2"] + "expect_skip_on_jobname": ["azure", "gcc", "msys2"] } diff --git a/test cases/frameworks/3 gmock/test.json b/test cases/frameworks/3 gmock/test.json index 6ace9de70..66afb97c3 100644 --- a/test cases/frameworks/3 gmock/test.json +++ b/test cases/frameworks/3 gmock/test.json @@ -1,3 +1,3 @@ { - "skip_on_jobname": ["azure", "cygwin", "macos", "msys2"] + "expect_skip_on_jobname": ["azure", "cygwin", "macos", "msys2"] } diff --git a/test cases/frameworks/30 scalapack/test.json b/test cases/frameworks/30 scalapack/test.json index 0c4057316..fe1016a0f 100644 --- a/test cases/frameworks/30 scalapack/test.json +++ b/test cases/frameworks/30 scalapack/test.json @@ -1,3 +1,3 @@ { - "skip_on_jobname": ["azure", "bionic", "cygwin", "fedora", "msys2", "opensuse"] + "expect_skip_on_jobname": ["azure", "bionic", "cygwin", "fedora", "msys2", "opensuse"] } diff --git a/test cases/frameworks/31 curses/test.json b/test cases/frameworks/31 curses/test.json index 03ed3a201..ed7c61929 100644 --- a/test cases/frameworks/31 curses/test.json +++ b/test cases/frameworks/31 curses/test.json @@ -3,10 +3,10 @@ "options": { "method": [ { "val": "pkg-config" }, - { "val": "config-tool", "skip_on_jobname": ["msys2"] }, - { "val": "system", "skip_on_os": ["windows"] } + { "val": "config-tool", "expect_skip_on_jobname": ["msys2"] }, + { "val": "system", "expect_skip_on_os": ["windows"] } ] } }, - "skip_on_jobname": ["azure", "cygwin"] + "expect_skip_on_jobname": ["azure", "cygwin"] } diff --git a/test cases/frameworks/34 gir static lib/test.json b/test cases/frameworks/34 gir static lib/test.json index 8d7f707d7..c2ba9b2a8 100644 --- a/test cases/frameworks/34 gir static lib/test.json +++ b/test cases/frameworks/34 gir static lib/test.json @@ -5,5 +5,5 @@ {"type": "file", "platform": "cygwin", "file": "usr/lib/libgirlib.dll.a"}, {"type": "file", "file": "usr/share/gir-1.0/Meson-1.0.gir"} ], - "skip_on_jobname": ["azure", "bionic", "cygwin", "macos", "msys2"] + "expect_skip_on_jobname": ["azure", "bionic", "cygwin", "macos", "msys2"] } diff --git a/test cases/frameworks/36 gtkdoc cpp/test.json b/test cases/frameworks/36 gtkdoc cpp/test.json index b2d9bc88f..c53e93cef 100644 --- a/test cases/frameworks/36 gtkdoc cpp/test.json +++ b/test cases/frameworks/36 gtkdoc cpp/test.json @@ -13,5 +13,5 @@ {"type": "file", "file": "usr/share/gtk-doc/html/foo/up.png"}, {"type": "file", "file": "usr/share/gtk-doc/html/foo/right.png"} ], - "skip_on_jobname": ["azure", "msys2"] + "expect_skip_on_jobname": ["azure", "msys2"] } diff --git a/test cases/frameworks/37 gir cpp/test.json b/test cases/frameworks/37 gir cpp/test.json index 3641d75ca..aa7f44e13 100644 --- a/test cases/frameworks/37 gir cpp/test.json +++ b/test cases/frameworks/37 gir cpp/test.json @@ -1,3 +1,3 @@ { - "skip_on_jobname": ["azure", "macos", "msys2", "cygwin"] + "expect_skip_on_jobname": ["azure", "macos", "msys2", "cygwin"] } diff --git a/test cases/frameworks/4 qt/test.json b/test cases/frameworks/4 qt/test.json index 4df0b49a0..34de94101 100644 --- a/test cases/frameworks/4 qt/test.json +++ b/test cases/frameworks/4 qt/test.json @@ -8,5 +8,5 @@ ] } }, - "skip_on_jobname": ["cygwin", "msys2", "azure"] + "expect_skip_on_jobname": ["cygwin", "msys2", "azure"] } diff --git a/test cases/frameworks/5 protocol buffers/test.json b/test cases/frameworks/5 protocol buffers/test.json index 6ace9de70..66afb97c3 100644 --- a/test cases/frameworks/5 protocol buffers/test.json +++ b/test cases/frameworks/5 protocol buffers/test.json @@ -1,3 +1,3 @@ { - "skip_on_jobname": ["azure", "cygwin", "macos", "msys2"] + "expect_skip_on_jobname": ["azure", "cygwin", "macos", "msys2"] } diff --git a/test cases/frameworks/6 gettext/test.json b/test cases/frameworks/6 gettext/test.json index 910fc1c74..5eaed9c24 100644 --- a/test cases/frameworks/6 gettext/test.json +++ b/test cases/frameworks/6 gettext/test.json @@ -22,5 +22,5 @@ ] } }, - "skip_on_jobname": ["azure", "cygwin"] + "expect_skip_on_jobname": ["azure", "cygwin"] } diff --git a/test cases/frameworks/7 gnome/test.json b/test cases/frameworks/7 gnome/test.json index f75ba131d..77c9bdfb5 100644 --- a/test cases/frameworks/7 gnome/test.json +++ b/test cases/frameworks/7 gnome/test.json @@ -36,5 +36,5 @@ {"type": "file", "file": "usr/include/simple-resources.h"}, {"type": "file", "file": "usr/include/generated-gdbus.h"} ], - "skip_on_jobname": ["azure", "cygwin", "macos", "msys2", "pypy"] + "expect_skip_on_jobname": ["azure", "cygwin", "macos", "msys2", "pypy"] } diff --git a/test cases/frameworks/8 flex/test.json b/test cases/frameworks/8 flex/test.json index 613cfe59f..083c31ab3 100644 --- a/test cases/frameworks/8 flex/test.json +++ b/test cases/frameworks/8 flex/test.json @@ -1,3 +1,3 @@ { - "skip_on_jobname": ["azure", "cygwin"] + "expect_skip_on_jobname": ["azure", "cygwin"] } |