add proxy for the first part and try to develop the second part
This commit is contained in:
Pierre Geeraert
@@ -0,0 +1,8 @@
|
||||
from ..objectmodel import ParseModel
|
||||
from .python import PythonCodeGenerator
|
||||
|
||||
|
||||
def codegen(model: ParseModel, parser_name: str = '') -> str:
|
||||
generator = PythonCodeGenerator(parser_name=parser_name)
|
||||
generator.walk(model)
|
||||
return generator.printed_text()
|
||||
@@ -0,0 +1,149 @@
|
||||
import builtins
|
||||
from collections import namedtuple
|
||||
|
||||
from .. import grammars, objectmodel
|
||||
from ..mixins.indent import IndentPrintMixin
|
||||
from ..util import compress_seq, safe_name
|
||||
from ..util.misc import topsort
|
||||
|
||||
HEADER = """\
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# WARNING: CAVEAT UTILITOR
|
||||
#
|
||||
# This file was automatically generated by TatSu.
|
||||
#
|
||||
# https://pypi.python.org/pypi/tatsu/
|
||||
#
|
||||
# Any changes you make to it will be overwritten the next time
|
||||
# the file is generated.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from dataclasses import dataclass
|
||||
|
||||
from tatsu.semantics import ModelBuilderSemantics
|
||||
{base_type_import}
|
||||
|
||||
|
||||
class {name}ModelBuilderSemantics(ModelBuilderSemantics):
|
||||
def __init__(self, context=None, types=None):
|
||||
types = [
|
||||
t for t in globals().values()
|
||||
if type(t) is type and issubclass(t, ModelBase)
|
||||
] + (types or [])
|
||||
super().__init__(context=context, types=types)
|
||||
"""
|
||||
|
||||
|
||||
BaseClassSpec = namedtuple('BaseClassSpec', ['class_name', 'base'])
|
||||
|
||||
|
||||
def modelgen(model: grammars.Grammar, name: str = '', base_type: type | None = objectmodel.Node) -> str:
|
||||
base_type = base_type or objectmodel.Node
|
||||
generator = PythonModelGenerator(name=name, base_type=base_type)
|
||||
return generator.generate_model(model)
|
||||
|
||||
|
||||
class PythonModelGenerator(IndentPrintMixin):
|
||||
|
||||
def __init__(self, name: str = '', base_type: type = objectmodel.Node):
|
||||
super().__init__()
|
||||
self.base_type = base_type
|
||||
self.name = name or None
|
||||
|
||||
def generate_model(self, grammar: grammars.Grammar):
|
||||
base_type = self.base_type
|
||||
base_type_name = base_type.__name__.split('.')[-1]
|
||||
base_type_import = f"from {base_type.__module__} import {base_type_name}"
|
||||
|
||||
self.name = self.name or grammar.name
|
||||
self.print(
|
||||
HEADER.format(
|
||||
name=self.name,
|
||||
base_type=self.base_type.__name__,
|
||||
base_type_import=base_type_import,
|
||||
),
|
||||
)
|
||||
|
||||
rule_index = {rule.name: rule for rule in grammar.rules}
|
||||
rule_specs = {
|
||||
rule.name: self._base_class_specs(rule)
|
||||
for rule in grammar.rules
|
||||
}
|
||||
rule_specs = {name: specs for name, specs in rule_specs.items() if specs}
|
||||
|
||||
specs_by_name = {
|
||||
s.class_name: s.base
|
||||
for specs in rule_specs.values()
|
||||
for s in specs
|
||||
}
|
||||
base = self._model_base_name()
|
||||
specs_by_name[base] = base_type_name
|
||||
|
||||
all_specs = {
|
||||
(s.class_name, s.base)
|
||||
for specs in rule_specs.values()
|
||||
for s in specs
|
||||
}
|
||||
model_names = topsort(reversed(specs_by_name), all_specs)
|
||||
|
||||
model_to_rule = {
|
||||
rule_specs[name][0].class_name: rule
|
||||
for name, rule in rule_index.items()
|
||||
if name in rule_specs
|
||||
}
|
||||
|
||||
for model_name in model_names:
|
||||
if model_name in vars(builtins):
|
||||
continue
|
||||
if rule := model_to_rule.get(model_name):
|
||||
self._gen_rule_class(rule, rule_specs[rule.name])
|
||||
else:
|
||||
self._gen_base_class(model_name, specs_by_name.get(model_name))
|
||||
|
||||
return self.printed_text()
|
||||
|
||||
@staticmethod
|
||||
def _model_base_name():
|
||||
return 'ModelBase'
|
||||
|
||||
def _gen_base_class(self, class_name: str, base: str | None):
|
||||
self.print()
|
||||
self.print()
|
||||
self.print('@dataclass(eq=False)')
|
||||
if base:
|
||||
self.print(f'class {class_name}({base}):')
|
||||
else:
|
||||
# FIXME: this cannot happen as base_type is the final base
|
||||
self.print(f'class {class_name}:')
|
||||
with self.indent():
|
||||
self.print('pass')
|
||||
|
||||
def _gen_rule_class(self, rule: grammars.Rule, specs: list[BaseClassSpec]):
|
||||
if not specs:
|
||||
return
|
||||
spec = specs[0]
|
||||
arguments = sorted({safe_name(d) for d, _ in compress_seq(rule.defines())})
|
||||
|
||||
self.print()
|
||||
self.print()
|
||||
self.print('@dataclass(eq=False)')
|
||||
self.print(f'class {spec.class_name}({spec.base}):')
|
||||
with self.indent():
|
||||
if not arguments:
|
||||
self.print('pass')
|
||||
for arg in arguments:
|
||||
self.print(f'{arg}: Any = None')
|
||||
|
||||
def _base_class_specs(self, rule: grammars.Rule) -> list[BaseClassSpec]:
|
||||
if not rule.params or not isinstance(rule.params[0], str):
|
||||
return []
|
||||
spec = rule.params[0].split('::')
|
||||
base = [self._model_base_name()]
|
||||
class_names = [safe_name(n) for n in spec] + base
|
||||
return [
|
||||
BaseClassSpec(class_name, class_names[i + 1])
|
||||
for i, class_name in enumerate(class_names[:-1])
|
||||
]
|
||||
@@ -0,0 +1,392 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import itertools
|
||||
import textwrap
|
||||
from collections.abc import Iterator
|
||||
from typing import Any
|
||||
|
||||
from .. import grammars
|
||||
from ..collections import OrderedSet as oset
|
||||
from ..exceptions import CodegenError
|
||||
from ..infos import UndefinedStr
|
||||
from ..mixins.indent import IndentPrintMixin
|
||||
from ..util import compress_seq, safe_name
|
||||
from ..walkers import NodeWalker
|
||||
|
||||
HEADER = """\
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# WARNING: CAVEAT UTILITOR
|
||||
#
|
||||
# This file was automatically generated by TatSu.
|
||||
#
|
||||
# https://pypi.python.org/pypi/tatsu/
|
||||
#
|
||||
# Any changes you make to it will be overwritten the next time
|
||||
# the file is generated.
|
||||
|
||||
# ruff: noqa: C405, COM812, I001, F401, PLR1702, PLC2801, SIM117
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from tatsu.buffering import Buffer
|
||||
from tatsu.parsing import Parser
|
||||
from tatsu.parsing import tatsumasu
|
||||
from tatsu.parsing import leftrec, nomemo, isname
|
||||
from tatsu.infos import ParserConfig
|
||||
from tatsu.util import re, generic_main
|
||||
"""
|
||||
|
||||
FOOTER = """\
|
||||
def main(filename, **kwargs):
|
||||
if not filename or filename == '-':
|
||||
text = sys.stdin.read()
|
||||
else:
|
||||
text = Path(filename).read_text()
|
||||
parser = {name}Parser()
|
||||
return parser.parse(
|
||||
text,
|
||||
filename=filename,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
import json
|
||||
from tatsu.util import asjson
|
||||
|
||||
ast = generic_main(main, {name}Parser, name='{name}')
|
||||
data = asjson(ast)
|
||||
print(json.dumps(data, indent=2))
|
||||
"""
|
||||
|
||||
|
||||
class PythonCodeGenerator(IndentPrintMixin, NodeWalker):
|
||||
_counter: Iterator[int] = itertools.count()
|
||||
|
||||
def __init__(self, parser_name: str = ''):
|
||||
super().__init__()
|
||||
self.parser_name = parser_name
|
||||
|
||||
@classmethod
|
||||
def _next_n(cls):
|
||||
return next(cls._counter)
|
||||
|
||||
@classmethod
|
||||
def _reset_counter(cls):
|
||||
cls._counter = itertools.count()
|
||||
|
||||
def walk_default(self, node: Any):
|
||||
return node
|
||||
|
||||
def walk_Grammar(self, grammar: grammars.Grammar):
|
||||
parser_name = self.parser_name or grammar.name
|
||||
self.print(HEADER)
|
||||
self.print()
|
||||
self.print()
|
||||
|
||||
self._gen_keywords(grammar)
|
||||
self._gen_buffering(grammar, parser_name)
|
||||
self._gen_parsing(grammar, parser_name)
|
||||
|
||||
self.print(FOOTER.format(name=parser_name))
|
||||
|
||||
def walk_Rule(self, rule: grammars.Rule):
|
||||
def param_repr(p):
|
||||
if isinstance(p, int | float):
|
||||
return str(p)
|
||||
else:
|
||||
return repr(p.split('::')[0])
|
||||
|
||||
self._reset_counter()
|
||||
params = kwparams = ''
|
||||
if rule.params:
|
||||
params = ', '.join(
|
||||
param_repr(self.walk(p)) for p in rule.params
|
||||
)
|
||||
if rule.kwparams:
|
||||
kwparams = ', '.join(
|
||||
f'{k}={param_repr(self.walk(v))}'
|
||||
for k, v in rule.kwparams.items()
|
||||
)
|
||||
|
||||
if params and kwparams:
|
||||
params = params + ', ' + kwparams
|
||||
elif kwparams:
|
||||
params = kwparams
|
||||
|
||||
leftrec = '\n@leftrec' if rule.is_leftrec else ''
|
||||
nomemo = (
|
||||
'\n@nomemo'
|
||||
if not rule.is_memoizable and not leftrec
|
||||
else ''
|
||||
)
|
||||
isname = '\n@isname' if rule.is_name else ''
|
||||
|
||||
self.print(
|
||||
f"""
|
||||
@tatsumasu({params})\
|
||||
{leftrec}\
|
||||
{nomemo}\
|
||||
{isname}\
|
||||
\ndef _{rule.name}_(self):
|
||||
""",
|
||||
)
|
||||
with self.indent():
|
||||
self.print(self.walk(rule.exp))
|
||||
|
||||
def walk_BasedRule(self, rule: grammars.BasedRule):
|
||||
# FIXME: the following override is to not alter the previous codegen
|
||||
rule.exp = rule.rhs
|
||||
self.walk_Rule(rule)
|
||||
|
||||
def walk_RuleRef(self, ref: grammars.RuleRef):
|
||||
self.print(f'self._{ref.name}_()')
|
||||
|
||||
def walk_RuleInclude(self, include: grammars.RuleInclude):
|
||||
self.walk(include.rule.exp)
|
||||
|
||||
def walk_Void(self, void: grammars.Void):
|
||||
self.print('self._void()')
|
||||
|
||||
def walk_Any(self, any: grammars.Any):
|
||||
self.print('self._any()')
|
||||
|
||||
def walk_Fail(self, fail: grammars.Fail):
|
||||
self.print('self._fail()')
|
||||
|
||||
def walk_Cut(self, cut: grammars.Cut):
|
||||
self.print('self._cut()')
|
||||
|
||||
def walk_Comment(self, comment: grammars.Comment):
|
||||
lines = '\n'.join(f'# {c!s}' for c in comment.comment.splitlines())
|
||||
self.print(f'\n{lines}\n')
|
||||
|
||||
def walk_EOLComment(self, comment: grammars.EOLComment):
|
||||
self.walk_Comment(comment)
|
||||
|
||||
def walk_EOF(self, eof: grammars.EOF):
|
||||
self.print('self._check_eof()')
|
||||
|
||||
def walk_Group(self, group: grammars.Group):
|
||||
self.print('with self._group():')
|
||||
with self.indent():
|
||||
self.walk(group.exp)
|
||||
|
||||
def walk_Token(self, token: grammars.Token):
|
||||
self.print(f'self._token({token.token!r})')
|
||||
|
||||
def walk_Constant(self, constant: grammars.Constant):
|
||||
self.print(f'self._constant({constant.literal!r})')
|
||||
|
||||
def walk_Alert(self, alert: grammars.Alert):
|
||||
self.print(f'self._alert({alert.literal!r}, {alert.level})')
|
||||
|
||||
def walk_Pattern(self, pattern: grammars.Pattern):
|
||||
self.print(f'self._pattern({pattern.pattern!r})')
|
||||
|
||||
def walk_Lookahead(self, lookahead: grammars.Lookahead):
|
||||
self.print('with self._if():')
|
||||
with self.indent():
|
||||
self.walk(lookahead.exp)
|
||||
|
||||
def walk_NegativeLookahead(self, lookahead: grammars.NegativeLookahead):
|
||||
self.print('with self._ifnot():')
|
||||
with self.indent():
|
||||
self.walk(lookahead.exp)
|
||||
|
||||
def walk_Sequence(self, seq: grammars.Sequence):
|
||||
self.walk(seq.sequence)
|
||||
self._gen_defines_declaration(seq)
|
||||
|
||||
def walk_Choice(self, choice: grammars.Choice):
|
||||
if len(choice.options) == 1:
|
||||
self.walk(choice.options[0])
|
||||
return
|
||||
|
||||
firstset = choice.lookahead_str()
|
||||
if firstset:
|
||||
msglines = textwrap.wrap(firstset, width=40)
|
||||
error = ['expecting one of: ', *msglines]
|
||||
else:
|
||||
error = ['no available options']
|
||||
errors = '\n'.join(repr(e) for e in error)
|
||||
|
||||
self.print('with self._choice():')
|
||||
with self.indent():
|
||||
self.walk(choice.options)
|
||||
self.print('self._error(')
|
||||
with self.indent():
|
||||
self.print(errors)
|
||||
self.print(')')
|
||||
|
||||
def walk_Option(self, option: grammars.Option):
|
||||
self.print('with self._option():')
|
||||
with self.indent():
|
||||
self.walk(option.exp)
|
||||
|
||||
def walk_Optional(self, optional: grammars.Optional):
|
||||
self.print('with self._optional():')
|
||||
with self.indent():
|
||||
self.walk(optional.exp)
|
||||
|
||||
def walk_EmptyClosure(self, closure: grammars.EmptyClosure):
|
||||
self.print('self._empty_closure()')
|
||||
|
||||
def walk_Closure(self, closure: grammars.Closure):
|
||||
n = self._gen_block(closure.exp)
|
||||
self.print(f'self._closure(block{n})')
|
||||
|
||||
def walk_PositiveClosure(self, closure: grammars.PositiveClosure):
|
||||
n = self._gen_block(closure.exp)
|
||||
self.print(f'self._positive_closure(block{n})')
|
||||
|
||||
def walk_Join(self, join: grammars.Join):
|
||||
m = self._gen_block(join.sep, name='sep')
|
||||
n = self._gen_block(join.exp)
|
||||
self.print(f'self._join(block{n}, sep{m})')
|
||||
|
||||
def walk_PositiveJoin(self, join: grammars.PositiveJoin):
|
||||
m = self._gen_block(join.sep, name='sep')
|
||||
n = self._gen_block(join.exp)
|
||||
self.print(f'self._positive_join(block{n}, sep{m})')
|
||||
|
||||
def walk_LeftJoin(self, join: grammars.LeftJoin):
|
||||
m = self._gen_block(join.sep, name='sep')
|
||||
n = self._gen_block(join.exp)
|
||||
self.print(f'self._left_join(block{n}, sep{m})')
|
||||
|
||||
def walk_RightJoin(self, join: grammars.RightJoin):
|
||||
m = self._gen_block(join.sep, name='sep')
|
||||
n = self._gen_block(join.exp)
|
||||
self.print(f'self._right_join(block{n}, sep{m})')
|
||||
|
||||
def walk_Gather(self, gather: grammars.Gather):
|
||||
m = self._gen_block(gather.sep, name='sep')
|
||||
n = self._gen_block(gather.exp)
|
||||
self.print(f'self._gather(block{n}, sep{m})')
|
||||
|
||||
def walk_PositiveGather(self, gather: grammars.PositiveGather):
|
||||
m = self._gen_block(gather.sep, name='sep')
|
||||
n = self._gen_block(gather.exp)
|
||||
self.print(f'self._positive_gather(block{n}, sep{m})')
|
||||
|
||||
def walk_SkipTo(self, skipto: grammars.SkipTo):
|
||||
n = self._gen_block(skipto.exp)
|
||||
self.print(f'self._skip_to(block{n})')
|
||||
|
||||
def walk_Named(self, named: grammars.Named):
|
||||
self.walk(named.exp)
|
||||
self.print(f"self.name_last_node('{named.name}')")
|
||||
|
||||
def walk_NamedList(self, named: grammars.Named):
|
||||
self.walk(named.exp)
|
||||
self.print(f"self.add_last_node_to_name('{named.name}')")
|
||||
|
||||
def walk_Override(self, override: grammars.Override):
|
||||
self.walk_Named(override)
|
||||
|
||||
def walk_OverrideList(self, override: grammars.OverrideList):
|
||||
self.walk_NamedList(override)
|
||||
|
||||
def walk_Special(self, special: grammars.Special):
|
||||
pass
|
||||
|
||||
def _gen_keywords(self, grammar: grammars.Grammar):
|
||||
keywords = [str(k) for k in grammar.keywords if k is not None]
|
||||
if not keywords:
|
||||
self.print('KEYWORDS: set[str] = set()')
|
||||
else:
|
||||
keywords_str = '\n'.join(f' {k!r},' for k in keywords)
|
||||
keywords_str = f'{{\n{keywords_str}\n}}'
|
||||
self.print(f'KEYWORDS: set[str] = {keywords_str}')
|
||||
|
||||
self.print()
|
||||
self.print()
|
||||
|
||||
def _gen_init(self, grammar: grammars.Grammar):
|
||||
start = grammar.config.start or grammar.rules[0].name
|
||||
|
||||
whitespace = grammar.config.whitespace
|
||||
if isinstance(whitespace, UndefinedStr): # the default value
|
||||
whitespace = None
|
||||
whitespace = repr(whitespace)
|
||||
|
||||
self.print(
|
||||
f'''
|
||||
config = ParserConfig.new(
|
||||
config,
|
||||
owner=self,
|
||||
whitespace={whitespace},
|
||||
nameguard={grammar.config.nameguard},
|
||||
ignorecase={grammar.config.ignorecase},
|
||||
namechars={grammar.config.namechars!r},
|
||||
parseinfo={grammar.config.parseinfo},
|
||||
comments={grammar.config.comments!r},
|
||||
eol_comments={grammar.config.eol_comments!r},
|
||||
keywords=KEYWORDS,
|
||||
start={start!r},
|
||||
)
|
||||
config = config.replace(**settings)
|
||||
''',
|
||||
)
|
||||
self.print()
|
||||
|
||||
def _gen_buffering(self, grammar: grammars.Grammar, parser_name: str):
|
||||
self.print(f'class {parser_name}Buffer(Buffer):')
|
||||
|
||||
with self.indent():
|
||||
self.print('def __init__(self, text, /, config: ParserConfig | None = None, **settings):')
|
||||
with self.indent():
|
||||
self._gen_init(grammar)
|
||||
self.print('super().__init__(text, config=config)')
|
||||
self.print()
|
||||
|
||||
def _gen_parsing(self, grammar: grammars.Grammar, parser_name: str):
|
||||
self.print()
|
||||
self.print(f'class {parser_name}Parser(Parser):')
|
||||
with self.indent():
|
||||
self.print('def __init__(self, /, config: ParserConfig | None = None, **settings):')
|
||||
with self.indent():
|
||||
self._gen_init(grammar)
|
||||
self.print('super().__init__(config=config)')
|
||||
self.print()
|
||||
self.walk(grammar.rules)
|
||||
self.print()
|
||||
|
||||
def _gen_defines_declaration(self, node: grammars.Model):
|
||||
defines = compress_seq(node.defines())
|
||||
ldefs = oset(safe_name(d) for d, value in defines if value)
|
||||
sdefs = oset(
|
||||
safe_name(d)
|
||||
for d, value in defines
|
||||
if not value and d not in ldefs
|
||||
)
|
||||
|
||||
if not (sdefs or ldefs):
|
||||
return
|
||||
|
||||
sdefs_str = ', '.join(sorted(repr(d) for d in sdefs))
|
||||
ldefs_str = ', '.join(sorted(repr(d) for d in ldefs))
|
||||
|
||||
if not ldefs:
|
||||
self.print(f'self._define([{sdefs_str}], [{ldefs_str}])')
|
||||
else:
|
||||
self.print('self._define(')
|
||||
with self.indent():
|
||||
self.print(f'[{sdefs_str}],')
|
||||
self.print(f'[{ldefs_str}],')
|
||||
self.print(')')
|
||||
|
||||
def _gen_block(self, exp: grammars.Model, name='block'):
|
||||
if () in exp.lookahead():
|
||||
raise CodegenError(f'{exp} may repeat empty sequence')
|
||||
|
||||
n = self._next_n()
|
||||
self.print()
|
||||
self.print(f'def {name}{n}():')
|
||||
with self.indent():
|
||||
self.walk(exp)
|
||||
|
||||
return n
|
||||
Reference in New Issue
Block a user