diff options
| -rw-r--r-- | docs/markdown/Codegen-module.md | 80 | ||||
| -rw-r--r-- | docs/markdown/_Sidebar.md | 1 | ||||
| -rw-r--r-- | docs/markdown/snippets/codegen_module.md | 3 | ||||
| -rw-r--r-- | docs/sitemap.txt | 1 | ||||
| -rw-r--r-- | docs/theme/extra/templates/navbar_links.html | 1 | ||||
| -rw-r--r-- | mesonbuild/modules/codegen.py | 274 | ||||
| -rwxr-xr-x | run_mypy.py | 1 | ||||
| -rw-r--r-- | test cases/frameworks/8 flex/meson.build | 24 |
8 files changed, 375 insertions, 10 deletions
diff --git a/docs/markdown/Codegen-module.md b/docs/markdown/Codegen-module.md new file mode 100644 index 000000000..5c2618d6b --- /dev/null +++ b/docs/markdown/Codegen-module.md @@ -0,0 +1,80 @@ +--- +short-description: Common Code Generators Module +authors: + - name: Dylan Baker + email: dylan@pnwbakers.com + years: [2024, 2025] +... + +# Codegen Module + +*(New in 1.10.0)* + +This module provides wrappers around common code generators, such as flex/lex. This purpose of this is to make it easier and more pleasant to use *common* code generators in a mesonic way. + +## Functions + +### lex() + +```meson +lex_gen = codegen.lex(implementations : ['flex', 'reflex'], flex_version : ['>= 2.6', '< 3'], reflex_version : '!= 1.4.2') +``` + +This function provides fine grained control over what implementation(s) and version(s) of lex are acceptable for a given project (These are set per-subproject). It returns a [new object](#lexgenerator), which can be used to generate code. + +It accepts the following keyword arguments: + + - `implementations`: a string array of acceptable implementations to use. May include: `lex`, `flex`, `reflex`, or `win_flex`. + - `lex_version`: a string array of version constraints to apply to the `lex` binary + - `flex_version`: a string array of version constraints to apply to the `flex` binary + - `reflex_version`: a string array of version constraints to apply to the `relex` binary + - `win_flex_version`: a string array of version constraints to apply to the `win_flex` binary + - `required`: A boolean or feature option + - `disabler`: Return a disabler if not found + - `native`: Is this generator for the host or build machine? + +## Returned Objects + +### LexGenerator + +#### lex.implementation + +```meson +lex = codegen.lex() +impl = lex.implementation() +``` + +Returns the string name of the lex implementation chosen. May be one of: + +- lex +- flex +- reflex +- win_flex + +#### lex.generate + +```meson +lex = codegen.lex() +lex.generate('lexer.l') +``` + +This function wraps flex, lex, reflex (but not RE/flex), and win_flex (on Windows). When using win_flex it will automatically add the `--wincompat` argument. + +This requires an input file, which may be a string, File, or generated source. It additionally takes the following options keyword arguments: + +- `args`: An array of extra arguments to pass the lexer +- `plainname`: If set to true then `@PLAINNAME@` will be used as the source base, otherwise `@BASENAME@`. +- `source`: the name of the source output. If this is unset Meson will use `{base}.{ext}` with an extension of `cpp` if the input has an extension of `.ll`, or `c` otherwise, with base being determined by the `plainname` argument. +- `header`: The optional output name for a header file. If this is unset no header is added +- `table`: The optional output name for a table file. If this is unset no table will be generated + +The outputs will be in the form `source [header] [table]`, which means those can be accessed by indexing the output of the `lex` call: + +```meson +lex = codegen.lex() +l1 = lex.generate('lexer.l', header : '@BASENAME@.h', table : '@BASENAME@.tab.h') +headers = [l1[1], l1[2]] # [header, table] + +l2 = lex.generate('lexer.l', table : '@BASENAME@.tab.h') +table = l2[1] +``` diff --git a/docs/markdown/_Sidebar.md b/docs/markdown/_Sidebar.md index ce73d5aad..eb6afb60f 100644 --- a/docs/markdown/_Sidebar.md +++ b/docs/markdown/_Sidebar.md @@ -9,6 +9,7 @@ ### [Modules](Module-reference.md) +* [codegen](Codegen-module.md) * [gnome](Gnome-module.md) * [i18n](i18n-module.md) * [pkgconfig](Pkgconfig-module.md) diff --git a/docs/markdown/snippets/codegen_module.md b/docs/markdown/snippets/codegen_module.md new file mode 100644 index 000000000..b4ea04c0a --- /dev/null +++ b/docs/markdown/snippets/codegen_module.md @@ -0,0 +1,3 @@ +## Experimental Codegen module + +A new module wrapping some common code generators has been added. Currently it supports lex/flex. diff --git a/docs/sitemap.txt b/docs/sitemap.txt index c8cb48cf1..b3967a27f 100644 --- a/docs/sitemap.txt +++ b/docs/sitemap.txt @@ -38,6 +38,7 @@ index.md Code-formatting.md Modules.md CMake-module.md + Codegen-module.md Cuda-module.md Dlang-module.md External-Project-module.md diff --git a/docs/theme/extra/templates/navbar_links.html b/docs/theme/extra/templates/navbar_links.html index 65a21a260..f16489896 100644 --- a/docs/theme/extra/templates/navbar_links.html +++ b/docs/theme/extra/templates/navbar_links.html @@ -7,6 +7,7 @@ <ul class="dropdown-menu" id="modules-menu"> @for tup in [ \ ("CMake-module.html","CMake"), \ + ("Codegen-module.html","Codegen"), \ ("Cuda-module.html","CUDA"), \ ("Dlang-module.html","Dlang"), \ ("External-Project-module.html","External Project"), \ diff --git a/mesonbuild/modules/codegen.py b/mesonbuild/modules/codegen.py new file mode 100644 index 000000000..b52f36f6d --- /dev/null +++ b/mesonbuild/modules/codegen.py @@ -0,0 +1,274 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright © 2024-2025 Intel Corporation + +from __future__ import annotations +import dataclasses +import os +import typing as T + +from . import ExtensionModule, ModuleInfo +from ..build import CustomTarget, CustomTargetIndex, GeneratedList +from ..compilers.compilers import lang_suffixes +from ..interpreter.interpreterobjects import extract_required_kwarg +from ..interpreter.type_checking import NoneType, REQUIRED_KW, DISABLER_KW, NATIVE_KW +from ..interpreterbase import ( + ContainerTypeInfo, ObjectHolder, KwargInfo, typed_pos_args, typed_kwargs, + noPosargs, noKwargs, disablerIfNotFound, InterpreterObject +) +from ..mesonlib import File, MesonException, Popen_safe +from ..programs import ExternalProgram, NonExistingExternalProgram +from ..utils.core import HoldableObject +from .. import mlog + +if T.TYPE_CHECKING: + from typing_extensions import Literal, TypeAlias, TypedDict + + from . import ModuleState + from .._typing import ImmutableListProtocol + from ..build import Executable + from ..interpreter import Interpreter + from ..interpreter.kwargs import ExtractRequired + from ..interpreterbase import TYPE_var, TYPE_kwargs + from ..mesonlib import MachineChoice + from ..programs import OverrideProgram + + Program: TypeAlias = T.Union[Executable, ExternalProgram, OverrideProgram] + LexImpls = Literal['lex', 'flex', 'reflex', 'win_flex'] + + class LexGenerateKwargs(TypedDict): + + args: T.List[str] + source: T.Optional[str] + header: T.Optional[str] + table: T.Optional[str] + plainname: bool + + class FindLexKwargs(ExtractRequired): + + lex_version: T.List[str] + flex_version: T.List[str] + reflex_version: T.List[str] + win_flex_version: T.List[str] + implementations: T.List[LexImpls] + native: MachineChoice + + +def is_subset_validator(choices: T.Set[str]) -> T.Callable[[T.List[str]], T.Optional[str]]: + + def inner(check: T.List[str]) -> T.Optional[str]: + if not set(check).issubset(choices): + invalid = ', '.join(sorted(set(check).difference(choices))) + valid = ', '.join(sorted(choices)) + return f"valid members are '{valid}', not '{invalid}'" + return None + + return inner + + +@dataclasses.dataclass +class _CodeGenerator(HoldableObject): + + name: str + program: Program + arguments: ImmutableListProtocol[str] = dataclasses.field(default_factory=list) + + def command(self) -> T.List[T.Union[Program, str]]: + return (T.cast('T.List[T.Union[Program, str]]', [self.program]) + + T.cast('T.List[T.Union[Program, str]]', self.arguments)) + + def found(self) -> bool: + return self.program.found() + + +@dataclasses.dataclass +class LexGenerator(_CodeGenerator): + pass + + +class LexHolder(ObjectHolder[LexGenerator]): + + @noPosargs + @noKwargs + @InterpreterObject.method('generate') + def implementation_method(self, args: T.List[TYPE_var], kwargs: TYPE_kwargs) -> str: + return self.held_object.name + + @noPosargs + @noKwargs + @InterpreterObject.method('found') + def found_method(self, args: T.List[TYPE_var], kwargs: TYPE_kwargs) -> bool: + return self.held_object.found() + + @typed_pos_args('codegen.lex.generate', (str, File, GeneratedList, CustomTarget, CustomTargetIndex)) + @typed_kwargs( + 'codegen.lex.generate', + KwargInfo('args', ContainerTypeInfo(list, str), default=[], listify=True), + KwargInfo('source', (str, NoneType)), + KwargInfo('header', (str, NoneType)), + KwargInfo('table', (str, NoneType)), + KwargInfo('plainname', bool, default=False), + ) + @InterpreterObject.method('generate') + def generate_method(self, args: T.Tuple[T.Union[str, File, GeneratedList, CustomTarget, CustomTargetIndex]], kwargs: LexGenerateKwargs) -> CustomTarget: + if not self.held_object.found(): + raise MesonException('Attempted to call generate without finding a lex implementation') + + input = self.interpreter.source_strings_to_files([args[0]])[0] + if isinstance(input, File): + is_cpp = input.endswith(".ll") + name = os.path.splitext(input.fname)[0] + else: + gen_input = input.get_outputs() + if len(gen_input) != 1: + raise MesonException('codegen.lex.generate: generated type inputs must have exactly one output, index into them to select the correct input') + is_cpp = gen_input[0].endswith('.ll') + name = os.path.splitext(gen_input[0])[0] + name = os.path.basename(name) + + # If an explicit source was given, use that to determine whether the + # user expects this to be a C or C++ source. + if kwargs['source'] is not None: + ext = kwargs['source'].rsplit('.', 1)[1] + is_cpp = ext in lang_suffixes['cpp'] + + for_machine = self.held_object.program.for_machine + + # Flex uses FlexLexer.h for C++ code + if is_cpp and self.held_object.name in {'flex', 'win_flex'}: + try: + comp = self.interpreter.environment.coredata.compilers[for_machine]['cpp'] + except KeyError: + raise MesonException(f"Could not find a C++ compiler for {for_machine} to search for FlexLexer.h") + found, _ = comp.has_header('FlexLexer.h', '', self.interpreter.environment) + if not found: + raise MesonException('Could not find FlexLexer.h, which is required for Flex with C++') + + if kwargs['source'] is None: + outputs = ['@{}@.{}'.format( + 'PLAINNAME' if kwargs['plainname'] else 'BASENAME', + 'cpp' if is_cpp else 'c')] + else: + outputs = [kwargs['source']] + + command = self.held_object.command() + if kwargs['header'] is not None: + outputs.append(kwargs['header']) + command.append(f'--header-file=@OUTPUT{len(outputs) - 1}@') + if kwargs['table'] is not None: + outputs.append(kwargs['table']) + command.append(f'--tables-file=@OUTPUT{len(outputs) - 1}@') + command.extend(kwargs['args']) + # Flex, at least, seems to require that input be the last argument given + command.append('@INPUT@') + + target = CustomTarget( + f'codegen-lex-{name}-{for_machine.get_lower_case_name()}', + self.interpreter.subdir, + self.interpreter.subproject, + self.interpreter.environment, + command, + [input], + outputs, + backend=self.interpreter.backend, + description='Generating lexer {{}} with {}'.format(self.held_object.name), + ) + self.interpreter.add_target(target.name, target) + + return target + + +class CodeGenModule(ExtensionModule): + + """Module with helpers for codegen wrappers.""" + + INFO = ModuleInfo('codegen', '1.10.0', unstable=True) + + def __init__(self, interpreter: Interpreter) -> None: + super().__init__(interpreter) + self.methods.update({ + 'lex': self.lex_method, + }) + + @noPosargs + @typed_kwargs( + 'codegen.lex', + KwargInfo('lex_version', ContainerTypeInfo(list, str), default=[], listify=True), + KwargInfo('flex_version', ContainerTypeInfo(list, str), default=[], listify=True), + KwargInfo('reflex_version', ContainerTypeInfo(list, str), default=[], listify=True), + KwargInfo('win_flex_version', ContainerTypeInfo(list, str), default=[], listify=True), + KwargInfo( + 'implementations', + ContainerTypeInfo(list, str), + default=[], + listify=True, + validator=is_subset_validator({'lex', 'flex', 'reflex', 'win_flex'}) + ), + REQUIRED_KW, + DISABLER_KW, + NATIVE_KW + ) + @disablerIfNotFound + def lex_method(self, state: ModuleState, args: T.Tuple, kwargs: FindLexKwargs) -> LexGenerator: + disabled, required, feature = extract_required_kwarg(kwargs, state.subproject) + if disabled: + mlog.log('generator lex skipped: feature', mlog.bold(feature), 'disabled') + return LexGenerator('lex', NonExistingExternalProgram('lex')) + + names: T.List[LexImpls] = [] + if kwargs['implementations']: + names = kwargs['implementations'] + else: + assert state.environment.machines[kwargs['native']] is not None, 'for mypy' + if state.environment.machines[kwargs['native']].system == 'windows': + names.append('win_flex') + names.extend(['flex', 'reflex', 'lex']) + + versions: T.Mapping[str, T.List[str]] = { + 'lex': kwargs['lex_version'], + 'flex': kwargs['flex_version'], + 'reflex': kwargs['reflex_version'], + 'win_flex': kwargs['win_flex_version'] + } + + for name in names: + bin = state.find_program( + name, wanted=versions[name], for_machine=kwargs['native'], required=False) + if bin.found(): + # If you're building reflex as a subproject, we consider that you + # know what you're doing. + if name == 'reflex' and isinstance(bin, ExternalProgram): + # there are potentially 3 programs called "reflex": + # 1. https://invisible-island.net/reflex/, an alternate fork + # of the original flex, this is supported + # 2. https://www.genivia.com/doc/reflex/html/, an + # alternative implementation for generating C++ scanners. + # Not supported + # 3. https://github.com/cespare/reflex, which is not a lex + # implementation at all, but a file watcher + _, out, err = Popen_safe(bin.get_command() + ['--version']) + if 'unknown flag: --version' in err: + mlog.debug('Skipping cespare/reflex, which is not a lexer and is not supported') + continue + if 'Written by Robert van Engelen' in out: + mlog.debug('Skipping RE/flex, which is not compatible with POSIX lex.') + continue + break + else: + if required: + raise MesonException.from_node( + 'Could not find a lex implementation. Tried: ', ", ".join(names), + node=state.current_node) + return LexGenerator(name, bin) + + lex_args: T.List[str] = [] + # This option allows compiling with MSVC + # https://github.com/lexxmark/winflexbison/blob/master/UNISTD_ERROR.readme + if bin.name == 'win_flex' and state.environment.machines[kwargs['native']].is_windows(): + lex_args.append('--wincompat') + lex_args.extend(['-o', '@OUTPUT0@']) + return LexGenerator(name, bin, T.cast('ImmutableListProtocol[str]', lex_args)) + + +def initialize(interpreter: Interpreter) -> CodeGenModule: + interpreter.append_holder_map(LexGenerator, LexHolder) + return CodeGenModule(interpreter) diff --git a/run_mypy.py b/run_mypy.py index 18c46f1f7..08f079aa8 100755 --- a/run_mypy.py +++ b/run_mypy.py @@ -52,6 +52,7 @@ modules = [ 'mesonbuild/msubprojects.py', 'mesonbuild/modules/__init__.py', 'mesonbuild/modules/cmake.py', + 'mesonbuild/modules/codegen.py', 'mesonbuild/modules/cuda.py', 'mesonbuild/modules/dlang.py', 'mesonbuild/modules/external_project.py', diff --git a/test cases/frameworks/8 flex/meson.build b/test cases/frameworks/8 flex/meson.build index 55b96dda7..070036072 100644 --- a/test cases/frameworks/8 flex/meson.build +++ b/test cases/frameworks/8 flex/meson.build @@ -1,10 +1,15 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright © 2024-2025 Intel Corporation + project('flex and bison', 'c') # The point of this test is that one generator # may output headers that are necessary to build # the sources of a different generator. -flex = find_program('flex', required: false) +# TODO: handle win_flex/win_bison + +flex = find_program('reflex', 'flex', 'lex', required: false) bison = find_program('bison', required: false) if not flex.found() @@ -15,11 +20,9 @@ if not bison.found() error('MESON_SKIP_TEST bison not found.') endif -lgen = generator(flex, -output : '@PLAINNAME@.yy.c', -arguments : ['-o', '@OUTPUT@', '@INPUT@']) - -lfiles = lgen.process('lexer.l') +codegen = import('unstable-codegen') +lex = codegen.lex(implementations : ['flex', 'reflex', 'lex']) +lfiles = lex.generate('lexer.l') pgen = generator(bison, output : ['@BASENAME@.tab.c', '@BASENAME@.tab.h'], @@ -27,10 +30,11 @@ arguments : ['@INPUT@', '--defines=@OUTPUT1@', '--output=@OUTPUT0@']) pfiles = pgen.process('parser.y') -e = executable('pgen', 'prog.c', - lfiles, - pfiles, - override_options: 'unity=off') +e = executable( + 'pgen', + 'prog.c', lfiles, pfiles, + override_options : ['unity=off'], +) test('parsertest', e, args: [meson.current_source_dir() / 'testfile']) |
