docs: Added Markdown generator

1 files changed, 405 insertions, 0 deletions

diff --git a/docs/refman/generatormd.py b/docs/refman/generatormd.py
new file mode 100644
index 000000000..831b6816f
--- /dev/null
+++ b/docs/refman/generatormd.py
@@ -0,0 +1,405 @@
+# Copyright 2021 The Meson development team
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .generatorbase import GeneratorBase
+import re
+
+from .model import (
+    ReferenceManual,
+    Function,
+    Method,
+    Object,
+    ObjectType,
+    Type,
+    DataTypeInfo,
+    ArgBase,
+    PosArg,
+    VarArgs,
+    Kwarg,
+)
+
+from pathlib import Path
+from textwrap import dedent
+import typing as T
+
+from mesonbuild import mlog
+
+PlaceholderTypes = T.Union[None, str, bool]
+FunctionDictType = T.Dict[
+    str,
+    T.Union[
+        PlaceholderTypes,
+        T.Dict[str, PlaceholderTypes],
+        T.Dict[str, T.Dict[str, PlaceholderTypes]],
+        T.Dict[str, T.List[T.Dict[str, PlaceholderTypes]]],
+        T.List[T.Dict[str, PlaceholderTypes]],
+    ]
+]
+
+_ROOT_BASENAME = 'RefMan'
+
+_OBJ_ID_MAP = {
+    ObjectType.ELEMENTARY: 'elementary',
+    ObjectType.BUILTIN: 'builtin',
+    ObjectType.MODULE: 'module',
+    ObjectType.RETURNED: 'returned',
+}
+
+# Indent all but the first line with 4*depth spaces.
+# This function is designed to be used with `dedent`
+# and fstrings where multiline strings are used during
+# the string interpolation.
+def smart_indent(raw: str, depth: int = 3) -> str:
+    lines = raw.split('\n')
+    first_line = lines[0]
+    lines = [' ' * (4 * depth) + x for x in lines]
+    lines[0] = first_line  # Do not indent the first line
+    return '\n'.join(lines)
+
+def code_block(code: str) -> str:
+    code = dedent(code)
+    return f'<pre><code class="language-meson">{code}</code></pre>'
+
+class GeneratorMD(GeneratorBase):
+    def __init__(self, manual: ReferenceManual, sitemap_out: Path, sitemap_in: Path) -> None:
+        super().__init__(manual)
+        self.sitemap_out = sitemap_out.resolve()
+        self.sitemap_in = sitemap_in.resolve()
+        self.out_dir = self.sitemap_out.parent
+        self.generated_files: T.Dict[str, str] = {}
+
+    # Utility functions
+    def _gen_filename(self, file_id: str, *, extension: str = 'md') -> str:
+        parts = file_id.split('.')
+        assert parts[0] == 'root'
+        assert all([x for x in parts])
+        parts[0] = _ROOT_BASENAME
+        parts = [re.sub(r'[0-9]+_', '', x) for x in parts]
+        return f'{"_".join(parts)}.{extension}'
+
+    def _object_from_ref(self, ref_str: str) -> T.Union[Function, Object]:
+        ids = ref_str.split('.')
+        ids = [x.strip() for x in ids]
+        assert len(ids) in [1, 2], f'Invalid object id "{ref_str}"'
+        assert not (ids[0].startswith('@') and len(ids) == 2), f'Invalid object id "{ref_str}"'
+        if ids[0].startswith('@'):
+            for obj in self.objects:
+                if obj.name == ids[0][1:]:
+                    return obj
+        if len(ids) == 2:
+            for obj in self.objects:
+                if obj.name != ids[0]:
+                    continue
+                for m in obj.methods:
+                    if m.name == ids[1]:
+                        return m
+                raise RuntimeError(f'Unknown method {ids[1]} in object {ids[0]}')
+            raise RuntimeError(f'Unknown object {ids[0]}')
+        for func in self.functions:
+            if func.name == ids[0]:
+                return func
+        raise RuntimeError(f'Unknown function or object {ids[0]}')
+
+    def _gen_object_file_id(self, obj: Object) -> str:
+        '''
+            Deterministically generate a unique file ID for the Object.
+
+            This ID determines where the object will be inserted in the sitemap.
+        '''
+        if obj.obj_type == ObjectType.RETURNED and obj.defined_by_module is not None:
+            base = self._gen_object_file_id(obj.defined_by_module)
+            return f'{base}.{obj.name}'
+        return f'root.{_OBJ_ID_MAP[obj.obj_type]}.{obj.name}'
+
+    def _link_to_object(self, obj: T.Union[Function, Object], text: T.Optional[str] = None) -> str:
+        '''
+            Generate a link to the function/method/object documentation.
+
+            The generated link is an HTML link (<a href="">text</a>) instead of
+            a Markdown link, so that the generated links can be used in custom
+            (or rather manual) code blocks.
+        '''
+        if isinstance(obj, Object):
+            text = text or f'<ins><code>{obj.name}</code></ins>'
+            link = self._gen_filename(self._gen_object_file_id(obj), extension="html")
+        elif isinstance(obj, Method):
+            text = text or f'<ins><code>`{obj.obj.name}.{obj.name}()`</code></ins>'
+            file = self._gen_filename(self._gen_object_file_id(obj.obj), extension="html")
+            link = f'{file}#{obj.obj.name}{obj.name}'
+        elif isinstance(obj, Function):
+            text = text or f'<ins><code>`{obj.name}()`</code></ins>'
+            link = f'{self._gen_filename("root.functions", extension="html")}#{obj.name}'
+        else:
+            raise RuntimeError(f'Invalid argument {obj}')
+        return f'<a href="{link}">{text}</a>'
+
+    def _write_file(self, data: str, file_id: str) -> None:#
+        ''' Write the data to disk.
+
+        Additionally, links of the form [[function]] are automatically replaced
+        with valid links to the correct URL. To reference objects / types use the
+        [[@object]] syntax.
+
+        Placeholders with the syntax [[!file_id]] will be replaced with the
+        corresponding generated markdown file.
+        '''
+        self.generated_files[file_id] = self._gen_filename(file_id)
+
+        # Replace [[func_name]] and [[obj.method_name]] with links
+        link_regex = re.compile(r'\[\[[^\]]+\]\]')
+        matches = link_regex.findall(data)
+        for i in matches:
+            obj_id: str = i[2:-2]
+            if obj_id.startswith('!'):
+                link_file_id = obj_id[1:]
+                data = data.replace(i, self._gen_filename(link_file_id))
+            else:
+                obj = self._object_from_ref(obj_id)
+                data = data.replace(i, self._link_to_object(obj))
+
+        out_file = self.out_dir / self.generated_files[file_id]
+        out_file.write_text(data, encoding='ascii')
+        mlog.log('Generated', mlog.bold(out_file.name))
+
+    def _write_template(self, data: T.Dict[str, T.Any], file_id: str, template_name: T.Optional[str] = None) -> None:
+        ''' Render the template mustache files and write the result '''
+        template_dir = Path(__file__).resolve().parent / 'templates'
+        template_name = template_name or file_id
+        template_name = f'{template_name}.mustache'
+        template_file = template_dir / template_name
+
+        # Import here, so that other generators don't also depend on it
+        import chevron
+        result = chevron.render(
+            template=template_file.read_text(encoding='utf-8'),
+            data=data,
+            partials_path=template_dir.as_posix(),
+            warn=True,
+        )
+
+        self._write_file(result, file_id)
+
+
+    # Actual generator functions
+    def _gen_func_or_method(self, func: Function) -> FunctionDictType:
+        def render_type(typ: Type) -> str:
+            def data_type_to_str(dt: DataTypeInfo) -> str:
+                base = self._link_to_object(dt.data_type, f'<ins>{dt.data_type.name}</ins>')
+                if dt.holds:
+                    return f'{base}[{render_type(dt.holds)}]'
+                return base
+            assert typ.resolved
+            return ' | '.join([data_type_to_str(x) for x in typ.resolved])
+
+        def len_stripped(s: str) -> int:
+            return len(re.sub(r'<[^>]+>', '', s))
+
+        def render_signature() -> str:
+            # Skip a lot of computations if the function does not take any arguments
+            if not any([func.posargs, func.optargs, func.kwargs, func.varargs]):
+                return f'{render_type(func.returns)} {func.name}()'
+
+            signature = dedent(f'''\
+                # {self.brief(func)}
+                {render_type(func.returns)} {func.name}(
+            ''')
+
+            # Calculate maximum lengths of the type and name
+            all_args: T.List[ArgBase] = []
+            all_args += func.posargs
+            all_args += func.optargs
+            all_args += [func.varargs] if func.varargs else []
+
+            max_type_len = 0
+            max_name_len = 0
+            if all_args:
+                max_type_len = max([len_stripped(render_type(x.type)) for x in all_args])
+                max_name_len = max([len(x.name) for x in all_args])
+
+            # Generate some common strings
+            def prepare(arg: ArgBase) -> T.Tuple[str, str, str, str]:
+                type_str = render_type(arg.type)
+                type_len = len_stripped(type_str)
+                type_space = ' ' * (max_type_len - type_len)
+                name_space = ' ' * (max_name_len - len(arg.name))
+                name_str = f'<b>{arg.name.replace("<", "&lt;").replace(">", "&gt;")}</b>'
+                return type_str, type_space, name_str, name_space
+
+            for i in func.posargs:
+                type_str, type_space, name_str, name_space = prepare(i)
+                signature += f'  {type_str}{type_space} {name_str},{name_space}     # {self.brief(i)}\n'
+
+            for i in func.optargs:
+                type_str, type_space, name_str, name_space = prepare(i)
+                signature += f'  {type_str}{type_space} [{name_str}],{name_space}   # {self.brief(i)}\n'
+
+            if func.varargs:
+                type_str, type_space, name_str, name_space = prepare(func.varargs)
+                signature += f'  {type_str}{type_space} {name_str}...,{name_space}  # {self.brief(func.varargs)}\n'
+
+            # Abort if there are no kwargs
+            if not func.kwargs:
+                return signature + ')'
+
+            # Only add this seperator if there are any posargs
+            if all_args:
+                signature += '\n  # Keyword arguments:\n'
+
+            # Recalculate lengths for kwargs
+            all_args = list(func.kwargs.values())
+            max_type_len = max([len_stripped(render_type(x.type)) for x in all_args])
+            max_name_len = max([len(x.name) for x in all_args])
+
+            for kwarg in self.sorted_and_filtered(list(func.kwargs.values())):
+                type_str, type_space, name_str, name_space = prepare(kwarg)
+                required = ' <i>[required]</i> ' if kwarg.required else '            '
+                required = required if any([x.required for x in func.kwargs.values()]) else ''
+                signature += f'  {name_str}{name_space} : {type_str}{type_space} {required} # {self.brief(kwarg)}\n'
+
+            return signature + ')'
+
+        def gen_arg_data(arg: T.Union[PosArg, Kwarg, VarArgs], *, optional: bool = False) -> T.Dict[str, PlaceholderTypes]:
+            data: T.Dict[str, PlaceholderTypes] = {
+                'name': arg.name,
+                'type': render_type(arg.type),
+                'description': arg.description,
+                'since': arg.since or None,
+                'deprecated': arg.deprecated or None,
+                'optional': optional,
+                'default': None,
+            }
+
+            if isinstance(arg, VarArgs):
+                data.update({
+                    'min': str(arg.min_varargs) if arg.min_varargs > 0 else '0',
+                    'max': str(arg.max_varargs) if arg.max_varargs > 0 else 'infinity',
+                })
+            if isinstance(arg, (Kwarg, PosArg)):
+                data.update({'default': arg.default or None})
+            if isinstance(arg, Kwarg):
+                data.update({'required': arg.required})
+            return data
+
+        mname = f'\\{func.name}' if func.name == '[index]' else func.name
+
+        data: FunctionDictType = {
+            'name': f'{func.obj.name}.{mname}' if isinstance(func, Method) else func.name,
+            'base_level': '##' if isinstance(func, Method) else '#',
+            'type_name_upper': 'Method' if isinstance(func, Method) else 'Function',
+            'type_name': 'method' if isinstance(func, Method) else 'function',
+            'description': func.description,
+            'notes': func.notes,
+            'warnings': func.warnings,
+            'example': func.example or None,
+            'signature_level': 'h4' if isinstance(func, Method) else 'h3',
+            'signature': render_signature(),
+            'has_args': bool(func.posargs or func.optargs or func.kwargs or func.varargs),
+            # Merge posargs and optargs by generating the *[optional]* tag for optargs
+            'posargs': {
+                'args': [gen_arg_data(x) for x in func.posargs] + [gen_arg_data(x, optional=True) for x in func.optargs]
+            } if func.posargs or func.optargs else None,
+            'kwargs':  {'args': [gen_arg_data(x) for x in self.sorted_and_filtered(list(func.kwargs.values()))]} if func.kwargs else None,
+            'varargs': gen_arg_data(func.varargs) if func.varargs else None,
+
+            # For the feature taggs template
+            'since': func.since or None,
+            'deprecated': func.deprecated or None,
+            'optional': False,
+            'default': None
+        }
+
+        return data
+
+    def _write_object(self, obj: Object) -> None:
+        data = {
+            'name': obj.name,
+            'description': obj.description,
+            'notes': obj.notes,
+            'warnings': obj.warnings,
+            'long_name': obj.long_name,
+            'obj_type_name': _OBJ_ID_MAP[obj.obj_type].capitalize(),
+            'example': obj.example or None,
+            'has_methods': bool(obj.methods),
+            'has_inherited_methods': bool(obj.inherited_methods),
+            'has_subclasses': bool(obj.extended_by),
+            'is_returned': bool(obj.returned_by),
+            'extends': obj.extends_obj.name if obj.extends_obj else None,
+            'returned_by': [self._link_to_object(x) for x in self.sorted_and_filtered(obj.returned_by)],
+            'extended_by': [self._link_to_object(x) for x in self.sorted_and_filtered(obj.extended_by)],
+            'methods': [self._gen_func_or_method(m) for m in self.sorted_and_filtered(obj.methods)],
+            'inherited_methods': [self._gen_func_or_method(m) for m in self.sorted_and_filtered(obj.inherited_methods)],
+        }
+
+        self._write_template(data, self._gen_object_file_id(obj), 'object')
+
+    def _write_functions(self) -> None:
+        data = {'functions': [self._gen_func_or_method(x) for x in self.functions]}
+        self._write_template(data, 'root.functions')
+
+    def _root_refman_docs(self) -> None:
+        def gen_obj_links(objs: T.List[Object]) -> T.List[T.Dict[str, str]]:
+            ret: T.List[T.Dict[str, str]] = []
+            for o in objs:
+                ret += [{'indent': '', 'link': self._link_to_object(o), 'brief': self.brief(o)}]
+                for m in self.sorted_and_filtered(o.methods):
+                    ret += [{'indent': '  ', 'link': self._link_to_object(m), 'brief': self.brief(m)}]
+                if o.obj_type == ObjectType.MODULE and self.extract_returned_by_module(o):
+                    tmp = gen_obj_links(self.extract_returned_by_module(o))
+                    tmp = [{**x, 'indent': '  ' + x['indent']} for x in tmp]
+                    ret += [{'indent': '  ', 'link': '**New objects:**', 'brief': ''}]
+                    ret += [*tmp]
+            return ret
+
+        data = {
+            'elementary': gen_obj_links(self.elementary),
+            'returned': gen_obj_links(self.returned),
+            'builtins': gen_obj_links(self.builtins),
+            'modules': gen_obj_links(self.modules),
+            'functions': [{'indent': '', 'link': self._link_to_object(x), 'brief': self.brief(x)} for x in self.functions],
+        }
+
+        self._write_template(data, 'root')
+        self._write_template({'name': 'Elementary types'}, f'root.{_OBJ_ID_MAP[ObjectType.ELEMENTARY]}', 'dummy')
+        self._write_template({'name': 'Builtin objects'},  f'root.{_OBJ_ID_MAP[ObjectType.BUILTIN]}',    'dummy')
+        self._write_template({'name': 'Returned objects'}, f'root.{_OBJ_ID_MAP[ObjectType.RETURNED]}',   'dummy')
+        self._write_template({'name': 'Modules'},          f'root.{_OBJ_ID_MAP[ObjectType.MODULE]}',     'dummy')
+
+
+    def generate(self) -> None:
+        mlog.log('Generating markdown files...')
+        with mlog.nested():
+            self._write_functions()
+            for obj in self.objects:
+                self._write_object(obj)
+            self._root_refman_docs()
+            self._configure_sitemap()
+
+    def _configure_sitemap(self) -> None:
+        '''
+            Replaces the `@REFMAN_PLACEHOLDER@` placeholder with the reference
+            manual sitemap. The structure of the sitemap is derived from the
+            file IDs.
+        '''
+        raw = self.sitemap_in.read_text(encoding='utf-8')
+        out = ''
+        for l in raw.split('\n'):
+            if '@REFMAN_PLACEHOLDER@' not in l:
+                out += f'{l}\n'
+                continue
+            mlog.log('Generating', mlog.bold(self.sitemap_out.as_posix()))
+            base_indent = l.replace('@REFMAN_PLACEHOLDER@', '')
+            for k in sorted(self.generated_files.keys()):
+                indent = base_indent + '\t' * k.count('.')
+                out += f'{indent}{self.generated_files[k]}\n'
+        self.sitemap_out.write_text(out, encoding='utf-8')