dependencies/dub: Correctly handle dub >= 1.35 as well as older dub

 - check version of DUB for compatibility with Meson
 - use "cacheArtifactPath" to locate DUB libraries in the cache
 - propose `dub build --deep` command to Meson users for missing DUB
   packages

Depending on DUB version, it will look either in the old cache structure
or use this new `dub describe` entry.

1 files changed, 157 insertions, 67 deletions

diff --git a/mesonbuild/dependencies/dub.py b/mesonbuild/dependencies/dub.py
index 0d25c3135..1c904ab2a 100644
--- a/mesonbuild/dependencies/dub.py
+++ b/mesonbuild/dependencies/dub.py
@@ -15,13 +15,59 @@ import json
 import typing as T
 
 if T.TYPE_CHECKING:
+    from typing_extensions import TypedDict
+
     from ..environment import Environment
 
+    # Definition of what `dub describe` returns (only the fields used by Meson)
+    class DubDescription(TypedDict):
+        platform: T.List[str]
+        architecture: T.List[str]
+        buildType: str
+        packages: T.List[DubPackDesc]
+        targets: T.List[DubTargetDesc]
+
+    class DubPackDesc(TypedDict):
+        name: str
+        version: str
+        active: bool
+        configuration: str
+        path: str
+        targetType: str
+        targetFileName: str
+
+    class DubTargetDesc(TypedDict):
+        rootPackage: str
+        linkDependencies: T.List[str]
+        buildSettings: DubBuildSettings
+        cacheArtifactPath: str
+
+    class DubBuildSettings(TypedDict):
+        importPaths: T.List[str]
+        stringImportPaths: T.List[str]
+        versions: T.List[str]
+        mainSourceFile: str
+        sourceFiles: T.List[str]
+        dflags: T.List[str]
+        libs: T.List[str]
+        lflags: T.List[str]
+
+    class FindTargetEntry(TypedDict):
+        search: str
+        artifactPath: str
 
 class DubDependency(ExternalDependency):
     # dub program and version
     class_dubbin: T.Optional[T.Tuple[ExternalProgram, str]] = None
     class_dubbin_searched = False
+    class_cache_dir = ''
+
+    # Map Meson Compiler ID's to Dub Compiler ID's
+    _ID_MAP: T.Mapping[str, str] = {
+        'dmd': 'dmd',
+        'gcc': 'gdc',
+        'llvm': 'ldc',
+    }
 
     def __init__(self, name: str, environment: 'Environment', kwargs: T.Dict[str, T.Any]):
         super().__init__(DependencyTypeName('dub'), environment, kwargs, language='d')
@@ -48,11 +94,20 @@ class DubDependency(ExternalDependency):
 
         assert isinstance(self.dubbin, ExternalProgram)
 
-        # Check if Dub version is compatible with Meson
-        if version_compare(dubver, '>1.31.1'):
+        # Check if Dub's compatibility with Meson
+        self._search_in_cache = version_compare(dubver, '<=1.31.1')
+        self._use_cache_describe = version_compare(dubver, '>=1.35.0')
+        self._dub_has_build_deep = version_compare(dubver, '>=1.35.0')
+
+        if not self._search_in_cache and not self._use_cache_describe:
             if self.required:
                 raise DependencyException(
-                    f"DUB version {dubver} is not compatible with Meson (can't locate artifacts in Dub cache)")
+                    f'DUB version {dubver} is not compatible with Meson'
+                    " (can't locate artifacts in DUB's cache). Upgrade to Dub >= 1.35.")
+            else:
+                mlog.warning(f'DUB dependency {name} not found because Dub {dubver} '
+                             "is not compatible with Meson. (Can't locate artifacts in DUB's cache)."
+                             ' Upgrade to Dub >= 1.35')
             self.is_found = False
             return
 
@@ -80,6 +135,20 @@ class DubDependency(ExternalDependency):
         elif dub_buildtype == 'minsize':
             dub_buildtype = 'release'
 
+        # A command that might be useful in case of missing DUB package
+        def dub_build_deep_command() -> str:
+            if self._dub_has_build_deep:
+                cmd = ['dub', 'build', '--deep']
+            else:
+                cmd = ['dub', 'run', '--yes', 'dub-build-deep', '--']
+
+            return join_args(cmd + [
+                main_pack_spec,
+                '--arch=' + dub_arch,
+                '--compiler=' + self.compiler.get_exelist()[-1],
+                '--build=' + dub_buildtype
+            ])
+
         # Ask dub for the package
         describe_cmd = [
             'describe', main_pack_spec, '--arch=' + dub_arch,
@@ -90,37 +159,31 @@ class DubDependency(ExternalDependency):
         if ret != 0:
             mlog.debug('DUB describe failed: ' + err)
             if 'locally' in err:
-                fetch_cmd = ['dub', 'fetch', main_pack_spec]
                 mlog.error(mlog.bold(main_pack_spec), 'is not present locally. You may try the following command:')
-                mlog.log(mlog.bold(join_args(fetch_cmd)))
+                mlog.log(mlog.bold(dub_build_deep_command()))
             self.is_found = False
             return
 
-        # A command that might be useful in case of missing DUB package
-        def dub_build_deep_command() -> str:
-            cmd = [
-                'dub', 'run', 'dub-build-deep', '--yes', '--', main_pack_spec,
-                '--arch=' + dub_arch, '--compiler=' + self.compiler.get_exelist()[-1],
-                '--build=' + dub_buildtype
-            ]
-            return join_args(cmd)
-
-        dub_comp_id = self.compiler.get_id().replace('llvm', 'ldc').replace('gcc', 'gdc')
-        description = json.loads(res)
+        dub_comp_id = self._ID_MAP[self.compiler.get_id()]
+        description: DubDescription = json.loads(res)
 
         self.compile_args = []
         self.link_args = self.raw_link_args = []
 
         show_buildtype_warning = False
 
-        def find_package_target(pkg: T.Dict[str, str]) -> bool:
+        # collect all targets
+        targets = {t['rootPackage']: t for t in description['targets']}
+
+        def find_package_target(pkg: DubPackDesc) -> bool:
             nonlocal show_buildtype_warning
             # try to find a static library in a DUB folder corresponding to
             # version, configuration, compiler, arch and build-type
             # if can find, add to link_args.
             # link_args order is meaningful, so this function MUST be called in the right order
             pack_id = f'{pkg["name"]}@{pkg["version"]}'
-            (tgt_file, compatibilities) = self._find_compatible_package_target(description, pkg, dub_comp_id)
+            tgt_desc = targets[pkg['name']]
+            (tgt_file, compatibilities) = self._find_target_in_cache(description, pkg, tgt_desc, dub_comp_id)
             if tgt_file is None:
                 if not compatibilities:
                     mlog.error(mlog.bold(pack_id), 'not found')
@@ -133,7 +196,7 @@ class DubDependency(ExternalDependency):
                     mlog.error(mlog.bold(pack_id), 'found but not compiled for', mlog.bold(dub_arch))
                 elif 'platform' not in compatibilities:
                     mlog.error(mlog.bold(pack_id), 'found but not compiled for',
-                               mlog.bold(description['platform'].join('.')))
+                               mlog.bold('.'.join(description['platform'])))
                 elif 'configuration' not in compatibilities:
                     mlog.error(mlog.bold(pack_id), 'found but not compiled for the',
                                mlog.bold(pkg['configuration']), 'configuration')
@@ -161,7 +224,7 @@ class DubDependency(ExternalDependency):
 
         # 1
         self.is_found = False
-        packages = {}
+        packages: T.Dict[str, DubPackDesc] = {}
         for pkg in description['packages']:
             packages[pkg['name']] = pkg
 
@@ -185,11 +248,6 @@ class DubDependency(ExternalDependency):
                 self.version = pkg['version']
                 self.pkg = pkg
 
-        # collect all targets
-        targets = {}
-        for tgt in description['targets']:
-            targets[tgt['rootPackage']] = tgt
-
         if name not in targets:
             self.is_found = False
             if self.pkg['targetType'] == 'sourceLibrary':
@@ -291,13 +349,22 @@ class DubDependency(ExternalDependency):
     # compiler, architecture, configuration...
     # It returns (target|None, {compatibilities})
     # If None is returned for target, compatibilities will list what other targets were found without full compatibility
-    def _find_compatible_package_target(self, jdesc: T.Dict[str, str], jpack: T.Dict[str, str], dub_comp_id: str) -> T.Tuple[str, T.Set[str]]:
-        dub_build_path = os.path.join(jpack['path'], '.dub', 'build')
+    def _find_target_in_cache(self, desc: DubDescription, pkg_desc: DubPackDesc,
+                              tgt_desc: DubTargetDesc, dub_comp_id: str
+                              ) -> T.Tuple[T.Optional[str], T.Set[str]]:
+        mlog.debug('Searching in DUB cache for compatible', pkg_desc['targetFileName'])
+
+        # recent DUB versions include a direct path to a compatible cached artifact
+        if self._use_cache_describe:
+            tgt_file = tgt_desc['cacheArtifactPath']
+            if os.path.exists(tgt_file):
+                return (tgt_file, {'configuration', 'platform', 'arch', 'compiler', 'compiler_version', 'build_type'})
+            else:
+                return (None, set())
 
-        if not os.path.exists(dub_build_path):
-            return (None, None)
+        assert self._search_in_cache
 
-        # try to find a dir like library-debug-linux.posix-x86_64-ldc_2081-EF934983A3319F8F8FF2F0E107A363BA
+        # try to find a string like library-debug-linux.posix-x86_64-ldc_2081-EF934983A3319F8F8FF2F0E107A363BA
 
         # fields are:
         #  - configuration
@@ -307,40 +374,16 @@ class DubDependency(ExternalDependency):
         #  - compiler id (dmd, ldc, gdc)
         #  - compiler version or frontend id or frontend version?
 
-        conf = jpack['configuration']
-        build_type = jdesc['buildType']
-        platforms = jdesc['platform']
-        archs = jdesc['architecture']
-
-        # Get D frontend version implemented in the compiler, or the compiler version itself
-        # gdc doesn't support this
-        comp_versions = []
-
-        if dub_comp_id != 'gdc':
-            comp_versions.append(self.compiler.version)
-
-            ret, res = self._call_compbin(['--version'])[0:2]
-            if ret != 0:
-                mlog.error('Failed to run {!r}', mlog.bold(dub_comp_id))
-                return (None, None)
-            d_ver_reg = re.search('v[0-9].[0-9][0-9][0-9].[0-9]', res)  # Ex.: v2.081.2
-
-            if d_ver_reg is not None:
-                frontend_version = d_ver_reg.group()
-                frontend_id = frontend_version.rsplit('.', 1)[0].replace(
-                    'v', '').replace('.', '')  # Fix structure. Ex.: 2081
-                comp_versions.extend([frontend_version, frontend_id])
-
-        compatibilities: T.Set[str] = set()
+        comp_versions = self._get_comp_versions_to_find(dub_comp_id)
 
         # build_type is not in check_list because different build types might be compatible.
         # We do show a WARNING that the build type is not the same.
         # It might be critical in release builds, and acceptable otherwise
-        check_list = ('configuration', 'platform', 'arch', 'compiler', 'compiler_version')
-
-        for entry in os.listdir(dub_build_path):
+        check_list = {'configuration', 'platform', 'arch', 'compiler', 'compiler_version'}
+        compatibilities: T.Set[str] = set()
 
-            target = os.path.join(dub_build_path, entry, jpack['targetFileName'])
+        for entry in self._cache_entries(pkg_desc):
+            target = entry['artifactPath']
             if not os.path.exists(target):
                 # unless Dub and Meson are racing, the target file should be present
                 # when the directory is present
@@ -350,33 +393,80 @@ class DubDependency(ExternalDependency):
             # we build a new set for each entry, because if this target is returned
             # we want to return only the compatibilities associated to this target
             # otherwise we could miss the WARNING about build_type
-            comps = set()
+            comps: T.Set[str] = set()
+
+            search = entry['search']
+
+            mlog.debug('searching compatibility in ' + search)
+            mlog.debug('compiler_versions', comp_versions)
 
-            if conf in entry:
+            if pkg_desc['configuration'] in search:
                 comps.add('configuration')
 
-            if build_type in entry:
+            if desc['buildType'] in search:
                 comps.add('build_type')
 
-            if all(platform in entry for platform in platforms):
+            if all(platform in search for platform in desc['platform']):
                 comps.add('platform')
 
-            if all(arch in entry for arch in archs):
+            if all(arch in search for arch in desc['architecture']):
                 comps.add('arch')
 
-            if dub_comp_id in entry:
+            if dub_comp_id in search:
                 comps.add('compiler')
 
-            if dub_comp_id == 'gdc' or any(cv in entry for cv in comp_versions):
+            if not comp_versions or any(cv in search for cv in comp_versions):
                 comps.add('compiler_version')
 
-            if all(key in comps for key in check_list):
+            if check_list.issubset(comps):
+                mlog.debug('Found', target)
                 return (target, comps)
             else:
                 compatibilities = set.union(compatibilities, comps)
 
         return (None, compatibilities)
 
+    def _cache_entries(self, pkg_desc: DubPackDesc) -> T.List[FindTargetEntry]:
+        # the "old" cache is the `.dub` directory in every package of ~/.dub/packages
+        dub_build_path = os.path.join(pkg_desc['path'], '.dub', 'build')
+
+        if not os.path.exists(dub_build_path):
+            mlog.warning('No such cache folder:', dub_build_path)
+            return []
+
+        mlog.debug('Checking in DUB cache folder', dub_build_path)
+
+        return [
+            {
+                'search': dir_entry,
+                'artifactPath': os.path.join(dub_build_path, dir_entry, pkg_desc['targetFileName'])
+            }
+            for dir_entry in os.listdir(dub_build_path)
+        ]
+
+    def _get_comp_versions_to_find(self, dub_comp_id: str) -> T.List[str]:
+        # Get D frontend version implemented in the compiler, or the compiler version itself
+        # gdc doesn't support this
+
+        if dub_comp_id == 'gdc':
+            return []
+
+        comp_versions = [self.compiler.version]
+
+        ret, res = self._call_compbin(['--version'])[0:2]
+        if ret != 0:
+            mlog.error('Failed to run', mlog.bold(' '.join(self.dubbin.get_command() + ['--version'])))
+            return []
+        d_ver_reg = re.search('v[0-9].[0-9][0-9][0-9].[0-9]', res)  # Ex.: v2.081.2
+
+        if d_ver_reg is not None:
+            frontend_version = d_ver_reg.group()
+            frontend_id = frontend_version.rsplit('.', 1)[0].replace(
+                'v', '').replace('.', '')  # Fix structure. Ex.: 2081
+            comp_versions.extend([frontend_version, frontend_id])
+
+        return comp_versions
+
     def _call_dubbin(self, args: T.List[str], env: T.Optional[T.Dict[str, str]] = None) -> T.Tuple[int, str, str]:
         assert isinstance(self.dubbin, ExternalProgram)
         p, out, err = Popen_safe(self.dubbin.get_command() + args, env=env)