add proxy for the first part and try to develop the second part

This commit is contained in:
Pierre Geeraert
2025-12-03 12:08:56 +00:00
parent 6c34d510e5
commit 460f18d85a
1095 changed files with 358799 additions and 5 deletions
@@ -0,0 +1,21 @@
from __future__ import annotations
from ._config import __toolname__, __version__
from .tool import ( # pylint: disable=W0622
compile,
main,
parse,
to_python_model,
to_python_sourcecode,
)
assert __version__
assert __toolname__
assert bool(compile)
assert bool(parse)
assert bool(to_python_sourcecode)
assert bool(to_python_model)
if __name__ == '__main__':
main()
@@ -0,0 +1,6 @@
from __future__ import annotations
import tatsu
if __name__ == '__main__':
tatsu.main()
@@ -0,0 +1,3 @@
from ._version import __version__ # noqa: F401
__toolname__ = 'TatSu'
@@ -0,0 +1 @@
__version__ = '5.13.1'
@@ -0,0 +1,134 @@
from __future__ import annotations
import copy
import operator
from functools import reduce
from .util import asjson, is_list
class AST(dict):
_frozen = False
def __init__(self, *args, **kwargs):
super().__init__()
self.update(*args, **kwargs)
self._frozen = True
@property
def frozen(self):
return self._frozen
@property
def parseinfo(self):
try:
return super().__getitem__('parseinfo')
except KeyError:
pass
def set_parseinfo(self, value):
super().__setitem__('parseinfo', value)
def copy(self):
return copy.copy(self)
def asjson(self):
return asjson(self)
def _set(self, key, value, force_list=False):
key = self._safekey(key)
previous = self.get(key)
if previous is None and force_list:
value = [value]
elif previous is None:
pass
elif is_list(previous):
value = [*previous, value]
else:
value = [previous, value]
super().__setitem__(key, value)
def _setlist(self, key, value):
return self._set(key, value, force_list=True)
def __copy__(self):
return AST(self)
def __getitem__(self, key):
if key in self:
return super().__getitem__(key)
key = self._safekey(key)
if key in self:
return super().__getitem__(key)
return None
def __setitem__(self, key, value):
self._set(key, value)
def __delitem__(self, key):
key = self._safekey(key)
super().__delitem__(key)
def __setattr__(self, name, value):
if self._frozen and name not in vars(self):
raise AttributeError(
f'{type(self).__name__} attributes are fixed. '
f' Cannot set attribute "{name}".',
)
super().__setattr__(name, value)
def __getattr__(self, name):
key = self._safekey(name)
if key in self:
return self[key]
elif name in self:
return self[name]
try:
return super().__getattribute__(name)
except AttributeError:
return None
def __hasattribute__(self, name):
try:
super().__getattribute__(name)
except (TypeError, AttributeError):
return False
else:
return True
def __reduce__(self):
return (AST, (list(self.items()),))
def _safekey(self, key):
while self.__hasattribute__(key):
key += '_'
return key
def _define(self, keys, list_keys=None):
for key in (self._safekey(k) for k in keys):
if key not in self:
super().__setitem__(key, None)
for key in (self._safekey(k) for k in list_keys or []):
if key not in self:
super().__setitem__(key, [])
def __json__(self, seen=None):
return {name: asjson(value, seen=seen) for name, value in self.items()}
def __repr__(self):
return repr(self.asjson())
def __str__(self):
return str(self.asjson())
def __hash__(self):
# NOTE: objects are actually mutable during creation
return reduce(
operator.xor,
(hash((name, id(value))) for name, value in self.items()),
0,
)
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,411 @@
"""
The Buffer class provides the functionality required by a parser-driven lexer.
Line analysis and caching are done so the parser can freely move with goto(p)
to any position in the parsed text, and still recover accurate information
about source lines and content.
"""
from __future__ import annotations
import re
from itertools import repeat, takewhile
from pathlib import Path
from typing import Any
from .exceptions import ParseError
from .infos import (
CommentInfo,
LineIndexInfo,
LineInfo,
ParserConfig,
PosLine,
UndefinedStr,
)
from .tokenizing import Tokenizer
from .util import (
contains_sublist,
extend_list,
identity,
)
from .util.misc import cached_re_compile, match_to_find
DEFAULT_WHITESPACE_RE = re.compile(r'(?m)\s+')
# for backwards compatibility with existing parsers
LineIndexEntry = LineIndexInfo
class Buffer(Tokenizer):
def __init__(
self, text, /, config: ParserConfig | None = None, **settings: Any,
):
super().__init__()
config = ParserConfig.new(config=config, owner=self, **settings)
self.config = config
text = str(text)
self.text = self.original_text = text
self.whitespace_re = self.build_whitespace_re(config.whitespace)
self.nameguard = (
config.nameguard
if config.nameguard is not None
else bool(self.whitespace_re) or bool(config.namechars)
)
self._namechar_set = set(config.namechars)
self._pos = 0
self._len = 0
self._linecount = 0
self._lines: list[str] = []
self._line_index: list[LineIndexInfo] = []
self._line_cache: list[PosLine] = []
self._comment_index: list[CommentInfo] = []
self._preprocess()
self._postprocess()
@property
def filename(self):
return self.config.filename
@property
def ignorecase(self):
return self.config.ignorecase
@property
def whitespace(self):
return self.config.whitespace
@staticmethod
def build_whitespace_re(whitespace):
if type(whitespace) is UndefinedStr:
return DEFAULT_WHITESPACE_RE
if whitespace in {None, ''}:
return None
elif isinstance(whitespace, re.Pattern):
return whitespace
elif whitespace:
return cached_re_compile(whitespace)
else:
return None
def _preprocess(self, *args, **kwargs):
lines, index = self._preprocess_block(self.filename, self.text)
self._lines = lines
self._line_index = index
self.text = self.join_block_lines(lines)
def _postprocess(self):
cache, count = PosLine.build_line_cache(self._lines)
self._line_cache = cache
self._linecount = count
self._len = len(self.text)
def _preprocess_block(self, name, block, **kwargs):
lines = self.split_block_lines(block)
index = LineIndexInfo.block_index(name, len(lines))
return self.process_block(name, lines, index, **kwargs)
def split_block_lines(self, block):
return block.splitlines(True)
def join_block_lines(self, lines):
return ''.join(lines)
def process_block(self, name, lines, index, **kwargs):
return lines, index
def include(self, lines, index, i, j, name, block, **kwargs):
blines, bindex = self._preprocess_block(name, block, **kwargs)
assert len(blines) == len(bindex)
lines[i:j] = blines
index[i:j] = bindex
assert len(lines) == len(index)
return j + len(blines) - 1
def include_file(self, source, name, lines, index, i, j):
text, filename = self.get_include(source, name)
return self.include(lines, index, i, j, filename, text)
def get_include(self, source, filename):
source = Path(source).resolve()
base = source.parent
include = base / filename
try:
with include.open() as f:
return f.read(), include
except OSError as e:
raise ParseError(f'include not found: {include}') from e
def replace_lines(self, i, j, name, block):
lines = self.split_block_lines(self.text)
index = list(self._line_index)
endline = self.include(lines, index, i, j, name, block)
self.text = self.join_block_lines(lines)
self._line_index = index
self._postprocess()
newtext = self.join_block_lines(lines[j + 1: endline + 2])
return endline, newtext
@property
def pos(self):
return self._pos
@pos.setter
def pos(self, p):
self.goto(p)
@property
def line(self):
return self.posline()
@property
def col(self):
return self.poscol()
def posline(self, pos=None):
if pos is None:
pos = self._pos
return self._line_cache[pos].line
def poscol(self, pos=None):
if pos is None:
pos = self._pos
start = self._line_cache[pos].start
return pos - start
def atend(self):
return self._pos >= self._len
def ateol(self):
return self.atend() or self.current in '\r\n'
@property
def current(self):
if self._pos >= self._len:
return None
return self.text[self._pos]
def at(self, p):
if p >= self._len:
return None
return self.text[p]
def peek(self, n=1):
return self.at(self._pos + n)
def next(self):
if self.atend():
return None
c = self.text[self._pos]
self._pos += 1
return c
def goto(self, pos):
self._pos = max(0, min(len(self.text), pos))
def move(self, n):
self.goto(self.pos + n)
def comments(self, p, clear=False):
if not self.config.comment_recovery or not self._comment_index:
return CommentInfo([], [])
n = self.posline(p)
if n >= len(self._comment_index):
return CommentInfo([], [])
eolcmm = []
if n < len(self._comment_index):
eolcmm = self._comment_index[n].eol
if clear:
self._comment_index[n].eol = []
cmm = []
while n >= 0 and self._comment_index[n].inline:
cmm.insert(0, self._comment_index[n].inline)
if clear:
self._comment_index[n].inline = []
n -= 1
return CommentInfo(cmm, eolcmm)
def _index_comments(self, comments, selector):
if comments and self.config.comment_recovery:
n = self.line
extend_list(
self._comment_index, n, default=CommentInfo.new_comment,
)
previous = selector(self._comment_index[n])
if not contains_sublist(
previous, comments,
): # FIXME: will discard repeated comments
previous.extend(comments)
def _eat_regex(self, regex):
if not regex:
return
while self._matchre_fast(regex):
pass
def _eat_regex_list(self, regex):
if not regex:
return []
regex = cached_re_compile(regex)
return list(takewhile(identity, map(self.matchre, repeat(regex))))
def eat_whitespace(self):
return self._eat_regex(self.whitespace_re)
def eat_comments(self):
comments = self._eat_regex_list(self.config.comments)
self._index_comments(comments, lambda x: x.inline)
def eat_eol_comments(self):
comments = self._eat_regex_list(self.config.eol_comments)
self._index_comments(comments, lambda x: x.eol)
def next_token(self):
p = None
while self._pos != p:
p = self._pos
self.eat_eol_comments()
self.eat_comments()
self.eat_whitespace()
def skip_to(self, c):
p = self._pos
le = self._len
while p < le and self.text[p] != c:
p += 1
self.goto(p)
return self.pos
def skip_past(self, c):
self.skip_to(c)
self.next()
return self.pos
def skip_to_eol(self):
return self.skip_to('\n')
def scan_space(self):
return (
self.whitespace_re and self._scanre(self.whitespace_re) is not None
)
def is_space(self):
return self.scan_space()
def is_name_char(self, c):
return c is not None and (c.isalnum() or c in self._namechar_set)
def match(self, token: str) -> str | None:
if token is None:
return self.atend()
p = self.pos
if self.ignorecase:
is_match = self.text[p: p + len(token)].lower() == token.lower()
else:
is_match = self.text[p: p + len(token)] == token
if not is_match:
return None
self.move(len(token))
partial_match = (
self.nameguard
and token
and token[0].isalpha()
and self.is_name_char(self.current)
and all(self.is_name_char(t) for t in token)
)
if partial_match:
self.goto(p)
return None
return token
def _matchre_fast(self, pattern):
if not (match := self._scanre(pattern)):
return
self.move(len(match.group()))
def matchre(self, pattern):
if not (match := self._scanre(pattern)):
return None
matched = match.group()
token = match_to_find(match)
self.move(len(matched))
return token
def _scanre(self, pattern):
cre = cached_re_compile(pattern)
return cre.match(self.text, self.pos)
@property
def linecount(self):
return self._linecount
def line_info(self, pos=None):
if pos is None:
pos = self._pos
# -2 to skip over sentinel
pos = min(pos, len(self._line_cache) - 2)
start, line, length = self._line_cache[pos]
end = start + length
col = pos - start
text = self.text[start:end]
# only required to support includes
n = min(len(self._line_index) - 1, line)
filename, line = self._line_index[n]
return LineInfo(filename, line, col, start, end, text)
def lookahead_pos(self):
if self.atend():
return ''
info = self.line_info()
return '~%d:%d' % (info.line + 1, info.col + 1)
def lookahead(self):
if self.atend():
return ''
info = self.line_info()
text = info.text[info.col: info.col + 1 + 80]
text = self.split_block_lines(text)[0].rstrip()
return f'{text}'
def get_line(self, n=None):
if n is None:
n = self.line
return self._lines[n]
def get_lines(self, start=None, end=None):
if start is None:
start = 0
if end is None:
end = len(self._lines)
return self._lines[start: end + 1]
def line_index(self, start=0, end=None):
if end is None:
end = len(self._line_index)
return self._line_index[start: 1 + end]
def __repr__(self):
return '%s@%d' % (type(self).__name__, self.pos)
def __json__(self, seen=None):
return None
@@ -0,0 +1,18 @@
from __future__ import annotations
from ..exceptions import CodegenError
from .cgbase import ( # noqa: F401
CodeGenerator,
DelegatingRenderingFormatter,
ModelRenderer,
NullModelRenderer,
)
def codegen(model, target='python'):
if target.lower() == 'python':
from tatsu.codegen import python
return python.codegen(model)
else:
raise CodegenError(f'Unknown target language: {target}')
@@ -0,0 +1,159 @@
from __future__ import annotations
from ..exceptions import CodegenError
from ..objectmodel import Node
from ..rendering import Renderer, RenderingFormatter, render
__all__ = [
'DelegatingRenderingFormatter',
'ModelRenderer',
'NullModelRenderer',
'CodeGenerator',
]
class DelegatingRenderingFormatter(RenderingFormatter):
def __init__(self, delegate):
assert hasattr(delegate, 'render')
super().__init__()
self.delegate = delegate
# override
def render(self, item, join='', **fields):
result = self.delegate.render(item, join=join, **fields)
if result is None:
result = super().render(item, join=join, **fields)
return result
def convert_field(self, value, conversion):
if isinstance(value, Node):
return self.render(value)
else:
return super().convert_field(value, conversion)
class ModelRenderer(Renderer):
def __init__(self, codegen, node, template=None):
super().__init__(template=template)
self._codegen = codegen
self._node = node
self.formatter = codegen.formatter
self.__postinit__()
def __postinit__(self):
pass
def __getattr__(self, name):
try:
super().__getattr__(name)
except AttributeError:
if name.startswith('_'):
raise
return getattr(self.node, name)
@property
def node(self):
return self._node
@property
def codegen(self):
return self._codegen
@property
def context(self):
return self._codegen
def get_renderer(self, item):
return self.codegen.get_renderer(item)
def render(self, **fields):
template = fields.pop('template', None)
if isinstance(self.node, Node):
fields.update(
{
k: v
for k, v in vars(self.node).items()
if not k.startswith('_')
},
)
else:
fields.update(value=self.node)
return super().render(template=template, **fields)
class NullModelRenderer(ModelRenderer):
"""A `ModelRenderer` that generates nothing."""
template = ''
class CodeGenerator:
"""
A **CodeGenerator** is an abstract class that finds a
``ModelRenderer`` class with the same name as each model's node and
uses it to render the node.
"""
def __init__(self, modules=None):
self.formatter = DelegatingRenderingFormatter(self)
self._renderers = {}
if modules is not None:
self._renderers = self._find_module_renderers(modules)
def _find_module_renderers(self, modules):
result = {}
for module in modules:
for name, dtype in vars(module).items():
if not isinstance(dtype, type):
continue
if not issubclass(dtype, ModelRenderer):
continue
if dtype is not ModelRenderer:
result[name] = dtype
return result
def _find_renderer_class(self, node):
if not isinstance(node, Node):
return None
node_class_name = node.__class__.__name__
classes = [node.__class__]
while classes:
cls = classes.pop()
name = cls.__name__
if name in self._renderers:
renderer = self._renderers[name]
self._renderers[node_class_name] = renderer
return renderer
for base in cls.__bases__:
if base not in classes:
classes.append(base)
raise CodegenError(f'Renderer for {node_class_name} not found')
def get_renderer(self, item):
if not isinstance(item, Node):
return None
renderer_class = self._find_renderer_class(item)
if renderer_class is None:
raise CodegenError(
f'Renderer not found for {type(item).__name__}',
)
try:
assert issubclass(renderer_class, ModelRenderer)
return renderer_class(self, item)
except Exception as e:
raise type(e)(str(e), renderer_class.__name__) from e
def render(self, item, join='', **fields):
renderer = self.get_renderer(item)
if renderer is None:
return render(item, join=join, **fields)
return str(renderer.render(**fields))
@@ -0,0 +1,233 @@
import inspect
from collections import namedtuple
from datetime import datetime
from tatsu.util import compress_seq, indent, re, safe_name
from ..codegen.cgbase import CodeGenerator, ModelRenderer
from ..exceptions import CodegenError
from ..objectmodel import BASE_CLASS_TOKEN, Node
from ..rendering import Renderer
NODE_NAME_PATTERN = r'(?!\d)\w+(' + rf'{BASE_CLASS_TOKEN}' + r'(?!\d)\w+)*'
TypeSpec = namedtuple('TypeSpec', ['class_name', 'base'])
DEFAULT_BASE_TYPE = """
@dataclass(eq=False)
class ModelBase(Node):
pass
"""
def codegen(model, base_type=None):
return ObjectModelCodeGenerator().render(model, base_type=base_type)
def _get_node_class_name(rule):
if not rule.params:
return None
typespec = rule.params[0]
if not isinstance(typespec, str):
return None
if not re.match(NODE_NAME_PATTERN, typespec):
return None
if not typespec[0].isupper():
return None
return typespec
def _typespec(rule):
if not _get_node_class_name(rule):
return []
spec = rule.params[0].split(BASE_CLASS_TOKEN)
class_names = [safe_name(n) for n in spec] + ['ModelBase']
typespec = []
for i, class_name in enumerate(class_names[:-1]):
base = class_names[i + 1]
typespec.append(TypeSpec(class_name, base))
return typespec
def _get_full_name(cls):
if not inspect.isclass(cls):
raise CodegenError('Base type has to be a class')
module = inspect.getmodule(cls)
if not module:
raise CodegenError('Base type has to be inside a module')
modulename = module.__name__
name = cls.__qualname__
# Try to reference the class
try:
idents = name.split('.')
cls_ = getattr(module, idents[0])
for ident in idents[1:]:
cls_ = getattr(cls_, ident)
assert cls_ == cls
except AttributeError as e:
raise CodegenError(
"Couldn't find base type, it has to be importable",
) from e
return modulename, name
class BaseTypeRenderer(Renderer):
def __init__(self, base_type):
super().__init__()
self.base_type = base_type
def render_fields(self, fields):
module, name = _get_full_name(self.base_type)
if '.' in name:
lookup = f'\nModelBase = {name}'
name = name.split('.')[0]
else:
lookup = ' as ModelBase'
fields.update(module=module, name=name, lookup=lookup)
template = """
from {module} import {name}{lookup}\
"""
class BaseClassRenderer(Renderer):
def __init__(self, spec):
super().__init__()
self.class_name = spec.class_name
self.base = spec.base
template = """
@dataclass(eq=False)
class {class_name}({base}):
pass\
"""
class ObjectModelCodeGenerator(CodeGenerator):
def _find_renderer_class(self, node):
if not isinstance(node, Node):
return None
name = node.__class__.__name__
renderer = globals().get(name)
if not renderer or not issubclass(renderer, ModelRenderer):
raise CodegenError(f'Renderer for {name} not found')
return renderer
class Rule(ModelRenderer):
def render_fields(self, fields):
defs = [safe_name(d) for d, _ in compress_seq(self.defines())]
defs = sorted(set(defs))
spec = fields['spec']
kwargs = '\n'.join(f'{d}: Any = None' for d in defs)
kwargs = indent(kwargs) if kwargs else indent('pass')
fields.update(
class_name=spec.class_name, base=spec.base, kwargs=kwargs,
)
template = """
@dataclass(eq=False)
class {class_name}({base}):
{kwargs}\
"""
class Grammar(ModelRenderer):
def render_fields(self, fields):
node_class_names = set()
bases = []
model_rules = []
for rule in self.node.rules:
specs = _typespec(rule)
if not specs:
continue
node_spec = specs[0]
base_spec = reversed(specs[1:])
if node_spec.class_name not in node_class_names:
model_rules.append((rule, node_spec))
bases.extend(
base
for base in base_spec
if base.class_name not in node_class_names
)
node_class_names.update(s.class_name for s in specs)
base_class_declarations = [
BaseClassRenderer(spec).render() for spec in bases
]
model_class_declarations = [
self.get_renderer(rule).render(spec=spec)
for rule, spec in model_rules
]
base_class_declarations = '\n\n\n'.join(base_class_declarations)
if base_class_declarations:
base_class_declarations += '\n\n'
model_class_declarations = '\n\n\n'.join(model_class_declarations)
version = datetime.now().strftime('%Y.%m.%d.%H')
base_type = fields['base_type']
fields.update(
name=self.node.name,
base_class_declarations=base_class_declarations,
model_class_declarations=model_class_declarations,
version=version,
base_type=BaseTypeRenderer(base_type).render()
if base_type
else DEFAULT_BASE_TYPE,
)
template = """\
#!/usr/bin/env python
# CAVEAT UTILITOR
#
# This file was automatically generated by TatSu.
#
# https://pypi.python.org/pypi/tatsu/
#
# Any changes you make to it will be overwritten the next time
# the file is generated.
from __future__ import annotations
from typing import Any
from dataclasses import dataclass
from tatsu.objectmodel import Node
from tatsu.semantics import ModelBuilderSemantics
{base_type}
class {name}ModelBuilderSemantics(ModelBuilderSemantics):
def __init__(self, context=None, types=None):
types = [
t for t in globals().values()
if type(t) is type and issubclass(t, ModelBase)
] + (types or [])
super().__init__(context=context, types=types)
{base_class_declarations}
{model_class_declarations}
"""
@@ -0,0 +1,594 @@
"""
Python code generation for models defined with tatsu.model
"""
from __future__ import annotations
import re
import textwrap
from .. import grammars
from ..collections import OrderedSet as oset
from ..exceptions import CodegenError
from ..objectmodel import BASE_CLASS_TOKEN, Node
from ..util import compress_seq, indent, safe_name, timestamp, trim
from .cgbase import CodeGenerator, ModelRenderer
class PythonCodeGenerator(CodeGenerator):
def _find_renderer_class(self, node):
if not isinstance(node, Node):
return None
name = node.__class__.__name__
renderer = globals().get(name)
if not renderer or not issubclass(renderer, Base):
raise CodegenError(f'Renderer for {name} not found')
return renderer
def codegen(model):
return PythonCodeGenerator().render(model)
class Base(ModelRenderer):
def defines(self):
return self.node.defines()
def make_defines_declaration(self):
defines = compress_seq(self.defines())
ldefs = oset(safe_name(d) for d, value in defines if value)
sdefs = oset(
safe_name(d)
for d, value in defines
if not value and d not in ldefs
)
if not (sdefs or ldefs):
return ''
else:
sdefs = '[{}]'.format(', '.join(sorted(repr(d) for d in sdefs)))
ldefs = '[{}]'.format(', '.join(sorted(repr(d) for d in ldefs)))
if not ldefs:
return f'\n\n self._define({sdefs}, {ldefs})'
else:
return '\n' + trim(self.define_template % (sdefs, ldefs))
define_template = """\
self._define(
%s,
%s,
)\
"""
class Void(Base):
template = 'self._void()'
class Any(Base):
template = 'self._any()'
class Fail(Base):
template = 'self._fail()'
class Comment(Base):
def render_fields(self, fields):
lines = '\n'.join(
f'# {c!s}' for c in self.node.comment.splitlines()
)
fields.update(lines=lines)
template = '\n{lines}\n'
class EOLComment(Comment):
pass
class EOF(Base):
template = 'self._check_eof()'
class _Decorator(Base):
template = '{exp}'
class Group(_Decorator):
template = """\
with self._group():
{exp:1::}\
"""
class Token(Base):
def render_fields(self, fields):
fields.update(token=repr(self.node.token))
template = 'self._token({token})'
class Constant(Base):
def render_fields(self, fields):
fields.update(constant=repr(self.node.literal))
template = 'self._constant({constant})'
class Alert(Base):
def render_fields(self, fields):
fields.update(literal=repr(self.node.literal), level=self.node.level)
template = 'self._alert({literal}, {level})'
class Pattern(Base):
def render_fields(self, fields):
raw_repr = repr(self.node.pattern)
fields.update(pattern=raw_repr)
template = 'self._pattern({pattern})'
class Lookahead(_Decorator):
template = """\
with self._if():
{exp:1::}\
"""
class NegativeLookahead(_Decorator):
template = """\
with self._ifnot():
{exp:1::}\
"""
class Sequence(Base):
def render_fields(self, fields):
fields.update(seq='\n'.join(self.rend(s) for s in self.node.sequence))
defines = self.make_defines_declaration()
fields.update(defines=defines)
template = """{seq}
{defines}\
"""
class Choice(Base):
def render_fields(self, fields):
firstset = self.node.lookahead_str()
if firstset:
msglines = textwrap.wrap(firstset, width=40)
error = ['expecting one of: ', *msglines]
else:
error = ['no available options']
error = [repr(e) for e in error]
fields.update(n=self.counter(), error=error)
fields.update(
options='\n'.join(self.rend(o) for o in self.node.options),
)
def render(self, **fields):
if len(self.node.options) == 1:
return self.rend(self.options[0], **fields)
else:
return super().render(**fields)
template = """\
with self._choice():
{options:1::}
self._error(
{error:2:\\n:} # noqa: COM812
)\
"""
class Option(_Decorator):
def render_fields(self, fields):
defines = self.make_defines_declaration()
fields.update(defines=defines)
template = """\
with self._option():
{exp:1::}\
"""
class Closure(_Decorator):
def render_fields(self, fields):
fields.update(n=self.counter())
def render(self, **fields):
if () in self.node.exp.lookahead():
raise CodegenError(f'{self.node} may repeat empty sequence')
return '\n' + super().render(**fields)
template = """\
def block{n}():
{exp:1::}
self._closure(block{n})\
"""
class PositiveClosure(Closure):
template = """\
def block{n}():
{exp:1::}
self._positive_closure(block{n})\
"""
class Join(_Decorator):
def render_fields(self, fields):
fields.update(n=self.counter())
def render(self, **fields):
if () in self.node.exp.lookahead():
raise CodegenError('may repeat empty sequence')
return '\n' + super().render(**fields)
template = """\
def sep{n}():
{sep:1::}
def block{n}():
{exp:1::}
self._join(block{n}, sep{n})\
"""
class PositiveJoin(Join):
template = """\
def sep{n}():
{sep:1::}
def block{n}():
{exp:1::}
self._positive_join(block{n}, sep{n})\
"""
class Gather(Join):
template = """\
def sep{n}():
{sep:1::}
def block{n}():
{exp:1::}
self._gather(block{n}, sep{n})\
"""
class PositiveGather(Join):
template = """\
def sep{n}():
{sep:1::}
def block{n}():
{exp:1::}
self._positive_gather(block{n}, sep{n})\
"""
class LeftJoin(PositiveJoin):
template = """\
def sep{n}():
{sep:1::}
def block{n}():
{exp:1::}
self._left_join(block{n}, sep{n})\
"""
class RightJoin(PositiveJoin):
template = """\
def sep{n}():
{sep:1::}
def block{n}():
{exp:1::}
self._right_join(block{n}, sep{n})\
"""
class EmptyClosure(Base):
template = 'self._empty_closure()'
class SkipTo(Closure):
template = """\
def block{n}():
{exp:1::}
self._skip_to(block{n})\
"""
class Optional(_Decorator):
template = """\
with self._optional():
{exp:1::}\
"""
class Cut(Base):
template = 'self._cut()'
class Named(_Decorator):
def __str__(self):
return f'{self.name}:{self.rend(self.exp)}'
def render_fields(self, fields):
fields.update(n=self.counter(), name=safe_name(self.node.name))
template = """
{exp}
self.name_last_node('{name}')\
"""
class NamedList(Named):
template = """
{exp}
self.add_last_node_to_name('{name}')\
"""
class Override(Named):
pass
class OverrideList(NamedList):
pass
class Special(Base):
pass
class RuleRef(Base):
template = 'self._{name}_()'
class RuleInclude(_Decorator):
def render_fields(self, fields):
super().render_fields(fields)
fields.update(exp=self.rend(self.node.rule.exp))
template = """
{exp}
"""
class Rule(_Decorator):
@staticmethod
def param_repr(p):
if isinstance(p, int | float):
return str(p)
else:
return repr(p.split(BASE_CLASS_TOKEN)[0])
def render(self, **fields):
try:
return super().render(**fields)
except CodegenError as e:
raise CodegenError(f'{self.node.name}={e}') from e
def render_fields(self, fields):
self.reset_counter()
params = kwparams = ''
if self.node.params:
params = ', '.join(
self.param_repr(self.rend(p)) for p in self.node.params
)
if self.node.kwparams:
kwparams = ', '.join(
f'{k}={self.param_repr(self.rend(v))}'
for k, v in self.kwparams.items()
)
if params and kwparams:
params = params + ', ' + kwparams
elif kwparams:
params = kwparams
fields.update(params=params)
sdefines = ''
if not isinstance(self.node, grammars.Choice):
sdefines = self.make_defines_declaration()
fields.update(defines=sdefines)
leftrec = self.node.is_leftrec
fields.update(leftrec='\n@leftrec' if leftrec else '')
fields.update(
nomemo='\n@nomemo'
if not self.node.is_memoizable and not leftrec
else '',
)
fields.update(isname='\n@isname' if self.node.is_name else '')
template = """
@tatsumasu({params})\
{leftrec}\
{nomemo}\
{isname}
def _{name}_(self):
{exp:1::}
"""
define_template = """\
self._define(
%s,
%s
)\
"""
class BasedRule(Rule):
def defines(self):
return self.rhs.defines()
def render_fields(self, fields):
super().render_fields(fields)
fields.update(exp=self.rhs)
class Grammar(Base):
def render_fields(self, fields):
abstract_template = trim(self.abstract_rule_template)
abstract_rules = [
abstract_template.format(name=safe_name(rule.name))
for rule in self.node.rules
]
abstract_rules = indent('\n'.join(abstract_rules))
whitespace = self.node.config.whitespace
if not whitespace:
whitespace = 'None'
elif isinstance(whitespace, re.Pattern):
whitespace = repr(whitespace)
else:
whitespace = f're.compile(r"{whitespace}")'
if self.node.config.nameguard is not None:
nameguard = repr(self.node.config.nameguard)
else:
nameguard = 'None'
ignorecase = self.node.config.ignorecase
left_recursion = self.node.config.left_recursion
parseinfo = self.node.config.parseinfo
namechars = repr(self.node.config.namechars or '')
comments = repr(self.node.config.comments)
eol_comments = repr(self.node.config.eol_comments)
rules = '\n'.join(
[self.get_renderer(rule).render() for rule in self.node.rules],
)
version = str(tuple(int(n) for n in str(timestamp()).split('.')))
keywords = [str(k) for k in self.keywords if k is not None]
keywords = '\n'.join(f' {k!r},' for k in keywords)
if keywords:
keywords = f'(\n{keywords}\n)'
fields.update(
rules=indent(rules),
start=self.node.rules[0].name,
abstract_rules=abstract_rules,
version=version,
whitespace=whitespace,
nameguard=nameguard,
ignorecase=ignorecase,
left_recursion=left_recursion,
parseinfo=parseinfo,
keywords=keywords,
namechars=namechars,
comments=comments,
eol_comments=eol_comments,
)
abstract_rule_template = """
def {name}(self, ast):
return ast
"""
template = """\
#!/usr/bin/env python3
# WARNING: CAVEAT UTILITOR
#
# This file was automatically generated by TatSu.
#
# https://pypi.python.org/pypi/tatsu/
#
# Any changes you make to it will be overwritten the next time
# the file is generated.
# ruff: noqa: C405, I001, F401, SIM117
import sys
from pathlib import Path
from tatsu.buffering import Buffer
from tatsu.parsing import Parser
from tatsu.parsing import tatsumasu
from tatsu.parsing import leftrec, nomemo, isname
from tatsu.infos import ParserConfig
from tatsu.util import re, generic_main
KEYWORDS: set[str] = set({keywords})
class {name}Buffer(Buffer):
def __init__(self, text, /, config: ParserConfig | None = None, **settings):
config = ParserConfig.new(
config,
owner=self,
whitespace={whitespace},
nameguard={nameguard},
ignorecase={ignorecase},
namechars={namechars},
parseinfo={parseinfo},
comments={comments},
eol_comments={eol_comments},
keywords=KEYWORDS,
start={start!r},
)
config = config.replace(**settings)
super().__init__(text, config=config)
class {name}Parser(Parser):
def __init__(self, /, config: ParserConfig | None = None, **settings):
config = ParserConfig.new(
config,
owner=self,
whitespace={whitespace},
nameguard={nameguard},
ignorecase={ignorecase},
namechars={namechars},
parseinfo={parseinfo},
comments={comments},
eol_comments={eol_comments},
left_recursion={left_recursion},
keywords=KEYWORDS,
start={start!r},
)
config = config.replace(**settings)
super().__init__(config=config)
{rules}
class {name}Semantics:
{abstract_rules}
def main(filename, **kwargs):
if not filename or filename == '-':
text = sys.stdin.read()
else:
text = Path(filename).read_text()
parser = {name}Parser()
return parser.parse(
text,
filename=filename,
**kwargs,
)
if __name__ == '__main__':
import json
from tatsu.util import asjson
ast = generic_main(main, {name}Parser, name='{name}')
data = asjson(ast)
print(json.dumps(data, indent=2))
"""
@@ -0,0 +1,2 @@
from .orderedset import OrderedSet # noqa: F401
from .tail import Tail # noqa: F401
@@ -0,0 +1,128 @@
# NOTE: from https://github.com/LuminosoInsight/ordered-set/blob/master/ordered_set.py
import itertools
from collections.abc import (
Iterable,
Iterator,
Mapping,
MutableSequence,
MutableSet,
Sequence,
)
from typing import Any, TypeVar
T = TypeVar('T')
class OrderedSet(MutableSet[T], Sequence[T]):
def __init__(self, iterable: Iterable[T] | None = None):
if iterable is not None:
self._map = dict.fromkeys(iterable)
else:
self._map = {}
self._list_cache: Sequence[T] | None = None
def __hash__(self):
return self._map.__hash__()
def __len__(self):
return len(self._map)
def __getitem__(self, i):
if self._list_cache is None:
self._list_cache = list(self._map.keys())
return self._list_cache[i]
def copy(self) -> 'OrderedSet[T]':
return self.__class__(self)
def __getstate__(self):
if len(self) == 0:
return (None,)
else:
return list(self)
def __setstate__(self, state):
if state == (None,):
self.__init__([])
else:
self.__init__(state)
def __contains__(self, key: Any) -> bool:
return key in self._map
def add(self, value: T): # pylint: disable=W0221
self._map[value] = len(self._map)
self._list_cache = None
def update(self, sequence: Iterable[T]):
self._map.update(dict.fromkeys(sequence))
self._list_cache = None
def pop(self) -> T:
key = next(iter(self._map.keys()))
self._map.pop(key)
self._list_cache = None
return key
def discard(self, value: T): # pylint: disable=W0221
self._map.pop(value, None)
self._list_cache = None
def clear(self):
self._map = {}
self._list_cache = None
def __iter__(self) -> Iterator[T]:
return iter(self._map.keys())
def __repr__(self) -> str:
return '{{{}}}'.format(', '.join(repr(e) for e in self))
def __eq__(self, other: Any) -> bool:
return all(item in other for item in self)
def union(self, *other: Iterable[T]) -> 'OrderedSet[T]':
# do not split `str`
outer = tuple(
[o] if not isinstance(o, set | Mapping | MutableSequence) else o
for o in other
)
inner = itertools.chain([self], *outer)
items = itertools.chain.from_iterable(inner)
return type(self)(itertools.chain(items))
def __and__(self, other: Iterable[Iterable[T]]) -> 'OrderedSet[T]':
return self.intersection(other)
def intersection(self, *other: Iterable[Iterable[T]]) -> 'OrderedSet[T]':
common = set.intersection(*other) # type: ignore[var-annotated, arg-type]
items = (item for item in self if item in common)
return type(self)(items)
def difference(self, *other: Iterable[T]) -> 'OrderedSet[T]':
other = set.union(*other) # type: ignore[assignment, arg-type]
items = (item for item in self if item not in other)
return type(self)(items)
def issubset(self, other: set[T]) -> bool:
return all(item in other for item in self)
def issuperset(self, other: set[T]) -> bool:
if len(self) < len(other): # Fast check for obvious cases
return False
return all(item in self for item in other)
def symmetric_difference(self, other: set[T]) -> 'OrderedSet[T]':
cls = type(self)
diff1 = cls(self).difference(other)
diff2 = cls(other).difference(self)
return cls().union(diff1, diff2)
def difference_update(self, *other: Iterable[T]):
self._map = dict.fromkeys(self.difference(*other))
def intersection_update(self, *other: Iterable[Iterable[T]]):
self._map = dict.fromkeys(self.intersection(*other))
def symmetric_difference_update(self, *other: Iterable[T]):
self._map = dict.fromkeys(self.difference(*other))
@@ -0,0 +1,37 @@
from collections import deque
from collections.abc import MutableSequence
class Tail(MutableSequence):
def __init__(self, maxlen):
self._tail = deque(maxlen=maxlen)
self._start = 0
@property
def start(self):
return self._start
def tailfull(self):
return len(self._tail) == self._tail.maxlen
def insert(self, index, value):
if self.tailfull():
self._tail.popleft()
self._start += 1
self._tail.insert(index - self._start, value)
def flush(self):
self._start += len(self._tail)
self._tail = deque(maxlen=self._tail.maxlen)
def __getitem__(self, index):
return self._tail[index - self._start]
def __setitem__(self, index, value):
self._tail[index - self._start] = value
def __delitem__(self, index):
del self._tail[index - self._start]
def __len__(self):
return len(self._tail) + self._start
@@ -0,0 +1,56 @@
# ruff: noqa: PLW0603
from __future__ import annotations
class _BF:
BLACK = ''
BLUE = ''
CYAN = ''
GREEN = ''
MAGENTA = ''
RED = ''
RESET = ''
WHITE = ''
YELLOW = ''
LIGHTBLACK_EX = ''
LIGHTRED_EX = ''
LIGHTGREEN_EX = ''
LIGHTYELLOW_EX = ''
LIGHTBLUE_EX = ''
LIGHTMAGENTA_EX = ''
LIGHTCYAN_EX = ''
LIGHTWHITE_EX = ''
class _Fore(_BF):
pass
class _Back(_BF):
pass
class _Style:
BRIGHT = ''
DIM = ''
NORMAL = ''
RESET_ALL = ''
Fore = _Fore
Back = _Back
Style = _Style
def init():
try:
import colorama
colorama.init()
except ImportError:
return
global Fore, Back, Style # pylint: disable=global-statement
Fore = colorama.Fore
Back = colorama.Back
Style = colorama.Style
@@ -0,0 +1,963 @@
from __future__ import annotations
import ast as stdlib_ast
import functools
import sys
from contextlib import contextmanager, suppress
from copy import copy
from . import buffering, color, tokenizing
from .ast import AST
from .collections import OrderedSet as oset
from .exceptions import (
FailedCut,
FailedExpectingEndOfText,
FailedKeywordSemantics,
FailedLeftRecursion,
FailedLookahead,
FailedParse,
FailedPattern,
FailedRef,
FailedSemantics,
FailedToken,
OptionSucceeded,
ParseError,
)
from .infos import (
Alert,
MemoKey,
ParseInfo,
ParserConfig,
ParseState,
RuleInfo,
RuleResult,
)
from .tokenizing import Tokenizer
from .util import (
info,
is_list,
left_assoc,
prune_dict,
right_assoc,
safe_name,
trim,
)
from .util.unicode_characters import (
C_CUT,
C_ENTRY,
C_FAILURE,
C_RECURSION,
C_SUCCESS,
)
__all__ = ['ParseContext', 'tatsumasu', 'leftrec', 'nomemo']
# decorator for rule implementation methods
def tatsumasu(*params, **kwparams):
def decorator(impl):
@functools.wraps(impl)
def wrapper(self):
name = impl.__name__
# remove the single leading and trailing underscore
# that the parser generator added
name = name[1:-1]
is_leftrec = getattr(impl, 'is_leftrec', False)
is_memoizable = getattr(impl, 'is_memoizable', True)
is_name = getattr(impl, 'is_name', False)
ruleinfo = RuleInfo(
name,
impl,
is_leftrec,
is_memoizable,
is_name,
params,
kwparams,
)
return self._call(ruleinfo)
return wrapper
return decorator
# This is used to mark left recursive rules
def leftrec(impl):
impl.is_leftrec = True
impl.is_memoizable = False
return impl
# Marks rules for which memoization has to be turned off
# (has no effect when left recursion is turned off)
def nomemo(impl):
impl.is_memoizable = False
return impl
# Marks rules marked as @name in the grammar
def isname(impl):
impl.is_name = True
return impl
class closure(list):
def __hash__(self):
return hash(tuple(self))
class ParseContext:
def __init__(self, /, config: ParserConfig | None = None, **settings):
super().__init__()
config = ParserConfig.new(config, **settings)
self.config = config
self._active_config = self.config
self._tokenizer: Tokenizer | None = None
self._semantics = config.semantics
self._initialize_caches()
def _initialize_caches(self):
self._statestack = [ParseState()]
self._rule_stack = []
self._cut_stack = [False]
self._last_node = None
self.substate = None
self._lookahead = 0
self._furthest_exception = None
self._clear_memoization_caches()
@property
def active_config(self):
return self._active_config
@property
def semantics(self):
return self._semantics
@property
def encoding(self):
return self.active_config.encoding
@property
def parseinfo(self):
return self.active_config.parseinfo
@property
def trace(self):
return self.active_config.trace
@property
def trace_length(self):
return self.active_config.trace_length
@property
def trace_separator(self):
return self.active_config.trace_separator
@property
def trace_filename(self):
return self.active_config.trace_filename
@property
def comments_re(self):
return self.active_config.comments_re
@property
def eol_comments_re(self):
return self.active_config.eol_comments_re
@property
def whitespace(self):
return self.active_config.whitespace
@property
def ignorecase(self):
return self.active_config.ignorecase
@property
def nameguard(self):
return self.active_config.nameguard
@property
def memoize_lookaheads(self):
return self.active_config.memoize_lookaheads
@property
def left_recursion(self):
return self.active_config.left_recursion
@property
def colorize(self):
return self.active_config.colorize
@property
def keywords(self):
return self._keywords
@property
def namechars(self):
return self.active_config.namechars
def _reset(self, config: ParserConfig):
if self.active_config.colorize:
color.init()
self._initialize_caches()
self._keywords = oset(config.keywords)
self._semantics = config.semantics
if hasattr(self.semantics, 'set_context'):
self.semantics.set_context(self)
return config
def _set_furthest_exception(self, e):
if (
not self._furthest_exception
or e.pos > self._furthest_exception.pos
):
self._furthest_exception = e
def parse(self, text, /, config: ParserConfig | None = None, **settings):
config = self.config.replace_config(config)
config = config.replace(**settings)
self._active_config = config
self._reset(config)
if isinstance(text, tokenizing.Tokenizer):
tokenizer = text
elif text is not None:
cls = self.tokenizercls
tokenizer = cls(text, config=config)
else:
raise ParseError('No tokenizer or text')
self._tokenizer = tokenizer
start = self.active_config.effective_rule_name() or 'start'
try:
rule = self._find_rule(start)
return rule()
except FailedCut as e:
self._set_furthest_exception(e.nested)
raise self._furthest_exception from e
except FailedParse as e:
self._set_furthest_exception(e)
raise self._furthest_exception from e
finally:
self._clear_memoization_caches()
@property
def tokenizer(self):
return self._tokenizer
@property
def tokenizercls(self) -> type[Tokenizer]:
if self.config.tokenizercls is None:
return buffering.Buffer
else:
return self.config.tokenizercls
@property
def last_node(self):
return self._last_node
@last_node.setter
def last_node(self, value):
self._last_node = value
@property
def _pos(self):
return self._tokenizer.pos
def _clear_memoization_caches(self):
self._memos = {}
self._results = {}
self._recursion_depth = 0
def _goto(self, pos):
self._tokenizer.goto(pos)
def _next(self):
return self._tokenizer.next()
def _next_token(self, ruleinfo=None):
if ruleinfo is None or not ruleinfo.name.lstrip('_')[:1].isupper():
self._tokenizer.next_token()
def _define(self, keys, list_keys=None):
# if self.ast and isinstance(self.ast, AST):
ast = AST()
ast._define(keys, list_keys=list_keys)
ast.update(self.ast)
self.ast = ast
@property
def state(self):
return self._statestack[-1]
@property
def ast(self):
return self.state.ast
@ast.setter
def ast(self, value):
self.state.ast = value
def name_last_node(self, name):
self.ast[name] = self.last_node
def add_last_node_to_name(self, name):
self.ast._setlist(name, self.last_node)
@staticmethod
def _safe_name(name, ast):
while name in ast:
name += '_'
return name
def ast_set(self, name, value, as_list=False):
ast = self.ast
name = self._safe_name(name, ast)
previous = ast.get(name)
if previous is None:
new_value = [value] if as_list else value
elif is_list(previous):
new_value = [*previous, value]
else:
new_value = [previous, value]
ast = AST(ast, name=new_value)
self.ast = ast
def ast_append(self, name, value):
self.ast_set(name, value, as_list=True)
def _push_ast(self, copyast=False):
ast = copy(self.ast) if copyast else AST()
self.state.pos = self._pos
self._statestack.append(ParseState(ast=ast, pos=self._pos))
def _pop_ast(self):
self._statestack.pop()
self.tokenizer.goto(self.state.pos)
def _merge_ast(self):
pos = self._pos
ast = self.ast
cst = self.cst
self._statestack.pop()
self.ast = ast
self._extend_cst(cst)
self.tokenizer.goto(pos)
@property
def cst(self):
return self.state.cst
@cst.setter
def cst(self, value):
self.state.cst = value
def _push_cst(self):
self._statestack.append(ParseState(ast=self.ast))
def _pop_cst(self):
ast = self.ast
self._statestack.pop()
self.ast = ast
def _merge_cst(self, extend=True):
cst = self.cst
self._pop_cst()
if extend:
self._extend_cst(cst)
else:
self._append_cst(cst)
return cst
def _append_cst(self, node):
self.last_node = node
previous = self.cst
if previous is None:
self.cst = self._copy_node(node)
elif is_list(previous):
previous.append(node)
else:
self.cst = [previous, node]
return node
def _extend_cst(self, node):
self.last_node = node
if node is None:
return None
previous = self.cst
if previous is None:
self.cst = self._copy_node(node)
elif is_list(node):
if is_list(previous):
previous.extend(node)
else:
self.cst = [previous, *node]
elif is_list(previous):
previous.append(node)
else:
self.cst = [previous, node]
return node
def _copy_node(self, node):
if node is None:
return None
elif is_list(node):
return node[:]
else:
return node
def _is_cut_set(self):
return self._cut_stack[-1]
def _cut(self):
self._trace_cut()
self._cut_stack[-1] = True
# Kota Mizushima et al say that we can throw away
# memos for previous positions in the tokenizer under
# certain circumstances, without affecting the linearity
# of PEG parsing.
# https://kmizu.github.io/papers/paste513-mizushima.pdf
#
# We adopt the heuristic of always dropping the cache for
# positions less than the current cut position. It remains to
# be proven if doing it this way affects linearity. Empirically,
# it hasn't.
def prune(cache, cutpos):
prune_dict(cache, lambda k, _: k[0] < cutpos)
prune(self._memos, self._pos)
def _memoization(self):
return self.memoize_lookaheads or self._lookahead == 0
def _rulestack(self):
rulestack = [r.name for r in reversed(self._rule_stack)]
stack = self.trace_separator.join(rulestack)
if max(len(s) for s in stack.splitlines()) > self.trace_length:
stack = stack[: self.trace_length]
stack = stack.rsplit(self.trace_separator, 1)[0]
stack += self.trace_separator
return stack
def _find_rule(self, name):
self._error(name, exclass=FailedRef)
return lambda: None # makes static checkers happy
def _find_semantic_action(self, name):
if self.semantics is None:
return None, None
postproc = getattr(self.semantics, '_postproc', None)
action = getattr(self.semantics, safe_name(name), None)
if not callable(action):
action = getattr(self.semantics, '_default', None)
if not callable(action):
action = None
if not callable(postproc):
postproc = None
return action, postproc
def _trace(self, msg, *params, **kwargs):
if self.trace:
msg %= params
info(str(msg), file=sys.stderr)
def _trace_event(self, event):
if self.trace:
fname = ''
if self.trace_filename:
fname = self._tokenizer.line_info().filename + '\n'
lookahead = self._tokenizer.lookahead().rstrip()
if lookahead:
lookahead = '\n' + lookahead
self._trace(
'%s %s%s%s',
event + self._rulestack(),
self._tokenizer.lookahead_pos(),
color.Style.DIM + fname,
color.Style.NORMAL + lookahead + color.Style.RESET_ALL,
end='',
)
def _trace_entry(self):
self._trace_event(color.Fore.YELLOW + color.Style.BRIGHT + C_ENTRY)
def _trace_success(self):
self._trace_event(color.Fore.GREEN + color.Style.BRIGHT + C_SUCCESS)
def _trace_failure(self, ex=None):
if isinstance(ex, FailedLeftRecursion):
self._trace_recursion()
else:
self._trace_event(color.Fore.RED + color.Style.BRIGHT + C_FAILURE)
def _trace_recursion(self):
self._trace_event(color.Fore.RED + color.Style.BRIGHT + C_RECURSION)
def _trace_cut(self):
self._trace_event(color.Fore.MAGENTA + color.Style.BRIGHT + C_CUT)
def _trace_match(self, token, name=None, failed=False):
if self.trace:
fname = ''
if self.trace_filename:
fname = self._tokenizer.line_info().filename + '\n'
name = f'/{name}/' if name else ''
if not failed:
fgcolor = color.Fore.GREEN + C_SUCCESS
else:
fgcolor = color.Fore.RED + C_FAILURE
lookahead = self._tokenizer.lookahead().rstrip()
if lookahead:
lookahead = '\n' + lookahead
self._trace(
color.Style.BRIGHT + fgcolor + "'%s' %s%s%s",
token,
name,
color.Style.DIM + fname,
color.Style.NORMAL + lookahead + color.Style.RESET_ALL,
end='',
)
def _make_exception(self, item, exclass=FailedParse):
rulestack = (r.name for r in self._rule_stack[::-1])
return exclass(self.tokenizer, rulestack, item)
def _error(self, item, exclass=FailedParse):
raise self._make_exception(item, exclass=exclass)
def _fail(self):
self._error('fail')
def _get_parseinfo(self, name, pos):
endpos = self._pos
return ParseInfo(
tokenizer=self.tokenizer,
rule=name,
pos=pos,
endpos=endpos,
line=self.tokenizer.posline(pos),
endline=self.tokenizer.posline(endpos),
alerts=self.state.alerts,
)
@property
def rule(self):
return self._rule_stack[-1]
@property
def memokey(self):
return MemoKey(self._pos, self.rule, self.substate)
def _memoize(self, key, memo):
if self._memoization() and key.rule.is_memoizable:
self._memos[key] = memo
return memo
def _forget(self, key):
self._memos.pop(key, None)
def _memo_for(self, key):
return self._memos.get(key)
# if isinstance(memo, FailedLeftRecursion):
# memo = self._results.get(key, memo)
def _mkresult(self, node):
return RuleResult(node, self._pos, self.substate)
def _save_result(self, key, result):
if is_list(result.node):
result = RuleResult(
closure(result.node), result.newpos, result.newstate,
)
self._results[key] = result
def _is_recursive(self, ruleinfo):
return ruleinfo.is_leftrec
def _set_left_recursion_guard(self, key):
ex = self._make_exception(key.rule.name, exclass=FailedLeftRecursion)
self._memoize(key, ex)
def _call(self, ruleinfo):
self._rule_stack += [ruleinfo]
pos = self._pos
try:
self._trace_entry()
self._last_node = None
result = self._recursive_call(ruleinfo)
node, newpos, newstate = result
self._goto(newpos)
self.substate = newstate
self._append_cst(node)
self._trace_success()
return node
except FailedPattern:
self._error(f'Expecting <{ruleinfo.name}>')
except FailedParse as e:
self._goto(pos)
self._set_furthest_exception(e)
self._trace_failure(e)
raise
finally:
self._rule_stack.pop()
def _clear_recursion_errors(self):
def filter(key, value):
return isinstance(value, FailedLeftRecursion)
prune_dict(self._memos, filter)
def _recursive_call(self, ruleinfo):
self._next_token(ruleinfo)
key = self.memokey
if not ruleinfo.is_leftrec:
return self._invoke_rule(ruleinfo, key)
elif not self.left_recursion:
self._error('Left recursion detected', exclass=FailedLeftRecursion)
self._recursion_depth += 1
if key in self._results:
result = self._results[key]
else:
result = self._make_exception(
ruleinfo.name, exclass=FailedLeftRecursion,
)
self._results[key] = result
initial = self._pos
lastpos = initial - 1
while True:
try:
self._clear_recursion_errors()
new_result = self._invoke_rule(ruleinfo, key)
self._goto(initial)
except FailedParse:
break
if new_result.newpos > lastpos:
self._save_result(key, new_result)
lastpos = new_result.newpos
result = new_result
else:
break
self._recursion_depth -= 1
if isinstance(result, Exception):
raise result # pylint: disable=raising-non-exception
return result
def _invoke_rule(self, ruleinfo, key):
memo = self._memo_for(key)
if isinstance(memo, Exception):
raise memo
if memo:
return memo
self._set_left_recursion_guard(key)
self._push_ast()
try:
try:
self._next_token(ruleinfo)
ruleinfo.impl(self)
node = self._get_node(key.pos, ruleinfo)
node = self._invoke_semantic_rule(ruleinfo, node)
result = self._mkresult(node)
self._memoize(key, result)
return result
except FailedSemantics as e:
self._error(str(e))
except FailedParse as e:
self._memoize(key, e)
raise
finally:
self._pop_ast()
def _get_node(self, pos, ruleinfo):
node = self.ast
if not node:
node = tuple(self.cst) if is_list(self.cst) else self.cst
elif '@' in node:
node = node['@'] # override the AST
elif self.parseinfo:
node.set_parseinfo(self._get_parseinfo(ruleinfo.name, pos))
return node
def _invoke_semantic_rule(self, rule, node):
semantic_rule, postproc = self._find_semantic_action(rule.name)
if semantic_rule:
node = semantic_rule(
node, *(rule.params or ()), **(rule.kwparams or {}),
)
if callable(postproc):
postproc(self, node) # pylint: disable=not-callable
if rule.is_name:
self._check_name(node)
return node
def _token(self, token):
self._next_token()
if self.tokenizer.match(token) is None:
self._trace_match(token, failed=True)
self._error(token, exclass=FailedToken)
self._trace_match(token)
self._append_cst(token)
return token
def _constant(self, literal):
self._next_token()
self._trace_match(literal)
if isinstance(literal, str):
try:
literal = stdlib_ast.literal_eval(literal.strip())
except (ValueError, SyntaxError):
if '\n' in literal:
literal = trim(literal)
literal = eval( # noqa: S307
f'{"f" + repr(literal)}', {}, self.ast,
)
self._append_cst(literal)
return literal
def _alert(self, message, level):
self._next_token()
self._trace_match(f'{"^" * level}`{message}`', failed=True)
self.state.alerts.append(Alert(message=message, level=level))
def _pattern(self, pattern):
token = self.tokenizer.matchre(pattern)
if token is None:
self._trace_match('', pattern, failed=True)
self._error(pattern, exclass=FailedPattern)
self._trace_match(token, pattern)
self._append_cst(token)
return token
def _eof(self):
return self.tokenizer.atend()
def _eol(self):
return self.tokenizer.ateol()
def _check_eof(self):
self._next_token()
if not self.tokenizer.atend():
self._error(
'Expecting end of text', exclass=FailedExpectingEndOfText,
)
@contextmanager
def _try(self):
s = self.substate
self._push_ast(copyast=True)
self.last_node = None
try:
yield
self._merge_ast()
except FailedParse:
self._pop_ast()
self.substate = s
raise
@contextmanager
def _option(self):
self.last_node = None
self._cut_stack += [False]
try:
with self._try():
yield
raise OptionSucceeded()
except FailedCut:
raise
except FailedParse as e:
if self._is_cut_set():
raise FailedCut(e) from e
finally:
self._cut_stack.pop()
@contextmanager
def _choice(self):
self.last_node = None
with suppress(OptionSucceeded):
yield
@contextmanager
def _optional(self):
self.last_node = None
with self._choice(), self._option():
yield
@contextmanager
def _group(self):
self._push_cst()
try:
yield
self._merge_cst(extend=True)
except Exception:
self._pop_cst()
raise
@contextmanager
def _if(self):
s = self.substate
self._push_ast()
self._lookahead += 1
try:
yield
finally:
self._pop_ast() # simply discard
self._lookahead -= 1
self.substate = s
@contextmanager
def _ifnot(self):
try:
with self._if():
yield
except FailedParse:
pass
else:
self._error('', exclass=FailedLookahead)
def _isolate(self, block, drop=False):
self._push_cst()
try:
block()
cst = self.cst
finally:
self._pop_cst()
if is_list(cst):
cst = closure(cst)
if not drop:
self._append_cst(cst)
return cst
def _repeat(self, block, prefix=None, dropprefix=False):
while True:
with self._choice():
with self._option():
p = self._pos
if prefix:
self._isolate(prefix, drop=dropprefix)
self._cut()
self._isolate(block)
if self._pos == p:
self._error('empty closure')
return
def _closure(self, block, sep=None, omitsep=False):
self._push_cst()
try:
self.cst = []
with self._optional():
block()
self.cst = [self.cst]
self._repeat(block, prefix=sep, dropprefix=omitsep)
self.cst = closure(self.cst)
return self._merge_cst()
except Exception:
self._pop_cst()
raise
def _positive_closure(self, block, sep=None, omitsep=False):
self._push_cst()
try:
block()
self.cst = [self.cst]
self._repeat(block, prefix=sep, dropprefix=omitsep)
self.cst = closure(self.cst)
return self._merge_cst()
except Exception:
self._pop_cst()
raise
def _empty_closure(self):
cst = closure([])
self._append_cst(cst)
return cst
def _gather(self, block, sep):
return self._closure(block, sep=sep, omitsep=True)
def _positive_gather(self, block, sep):
return self._positive_closure(block, sep=sep, omitsep=True)
def _join(self, block, sep):
return self._closure(block, sep=sep)
def _positive_join(self, block, sep):
return self._positive_closure(block, sep=sep)
def _left_join(self, block, sep):
self.cst = left_assoc(self._positive_join(block, sep))
self.last_node = self.cst
return self.cst
def _right_join(self, block, sep):
self.cst = right_assoc(self._positive_join(block, sep))
self.last_node = self.cst
return self.cst
def _check_name(self, name):
name = str(name)
if self.ignorecase or self.tokenizer.ignorecase:
name = name.upper()
if name in self.keywords:
raise FailedKeywordSemantics(f'"{name}" is a reserved word')
def _void(self):
self.last_node = None
def _any(self):
c = self._next()
if c is None:
self._trace_match(c, failed=True)
self._error(c, exclass=FailedToken)
self._trace_match(c)
self._append_cst(c)
return c
def _skip_to(self, block):
while not self._eof():
try:
with self._if():
block()
except FailedParse:
pass
else:
break
pos = self._pos
self._next_token()
if self._pos == pos:
self._next()
block()
@@ -0,0 +1,282 @@
from __future__ import annotations
import itertools
import pygraphviz as pgv # pylint: disable=E0401
from .walkers import NodeWalker
__all__ = ['draw']
def draw(filename, grammar):
traverser = GraphvizWalker()
traverser.walk(grammar)
traverser.draw(filename)
class GraphvizWalker(NodeWalker):
def __init__(self):
super().__init__()
self.top_graph = pgv.AGraph(
directed=True, rankdir='LR', packMode='clust', splines='true',
)
self.stack = [self.top_graph]
self.node_count = 0
@property
def graph(self):
return self.stack[-1]
def draw(self, filename):
self.graph.layout(prog='dot')
# WARNING: neato generated graphics hang my GPU
# self.graph.layout(prog='neato')
self.graph.draw(filename)
def push_graph(self, name=None, **attr):
if name is None:
self.node_count += 1
name = 'g%d' % self.node_count
self.stack.append(self.graph.add_subgraph(name, **attr))
return self.graph
def pop_graph(self):
self.stack.pop()
def node(self, name, id=None, **attr):
if id is None:
self.node_count += 1
id = 'n%d' % self.node_count
else:
try:
return self.graph.get_node(id)
except KeyError:
pass
self.graph.add_node(id, **attr)
n = self.graph.get_node(id)
n.attr['label'] = name
# n.attr['shape'] = 'circle'
return n
def tnode(self, name, **attr):
return self.node(name, **attr)
def dot(self):
n = self.node('')
n.attr['shape'] = 'point'
n.attr['size'] = 0.0000000001
n.attr['label'] = ''
return n
def start_node(self):
return self.dot()
def ref_node(self, name):
n = self.node(name)
n.attr['shape'] = 'box'
return n
def rule_node(self, name, **attr):
n = self.node(name, **attr)
n.attr['shape'] = 'parallelogram'
return n
def end_node(self):
n = self.node('')
n.attr['shape'] = 'point'
n.attr['width'] = 0.1
return n
def edge(self, s, e, **attr):
self.graph.add_edge(s, e, **attr)
edge = self.graph.get_edge(s, e)
# edge.attr['arrowhead'] = 'normal'
edge.attr['arrowhead'] = 'none'
return edge
def redge(self, s, e):
edge = self.edge(s, e)
edge.attr['dir'] = 'back'
return edge
def zedge(self, s, e):
return self.edge(s, e, len=0.000001)
def nedge(self, s, e):
return self.edge(s, e, style='invisible', dir='none')
def path(self, p):
self.graph.add_path(p)
def subgraph(self, name, bunch):
self.top_graph.add_subgraph(name)
def concat(self, *args):
return list(itertools.chain(*args))
def _walk_decorator(self, d):
return self.walk(d.exp)
def walk_default(self, node):
raise NotImplementedError('No walking for ', type(node).__name__)
def walk__decorator(self, d):
return self.walk(d.exp)
def walk__grammar(self, g):
self.push_graph(g.name + '0')
try:
vrules = [self.walk(r) for r in reversed(g.rules)]
finally:
self.pop_graph()
self.push_graph(g.name + '1')
try:
pass
# link all rule starting nodes with invisible edges
# starts = [self.node(r.name, id=r.name) for r in g.rules]
# for n1, n2 in zip(starts, starts[1:]):
# self.nedge(n1, n2)
finally:
self.pop_graph()
s, t = vrules[0][0], vrules[-1][1]
return (s, t)
def walk__rule(self, r):
self.push_graph(r.name)
try:
i, e = self.walk(r.exp)
s = self.rule_node(r.name, id=r.name)
self.edge(s, i)
t = self.end_node()
self.edge(e, t)
return (s, t)
finally:
self.pop_graph()
def walk__based_rule(self, r):
return self.walk__rule(r)
def walk__rule_ref(self, rr):
n = self.ref_node(rr.name)
return (n, n)
def walk__special(self, s):
n = self.node(s.special)
return (n, n)
def walk__override(self, o):
return self._walk_decorator(o)
def walk__named(self, n):
return self._walk_decorator(n)
def walk__named_list(self, n):
return self._walk_decorator(n)
def walk__cut(self, c):
# c = self.node('>>')
# return (c, c)
return None
def walk__optional(self, o):
i, e = self._walk_decorator(o)
ni = self.dot()
ne = self.dot()
self.zedge(ni, i)
self.edge(ni, ne)
self.zedge(e, ne)
return (ni, ne)
def walk__closure(self, r):
self.push_graph(rankdir='TB')
try:
i, e = self._walk_decorator(r)
ni = self.dot()
self.edge(ni, i)
self.edge(e, ni)
return (ni, ni)
finally:
self.pop_graph()
def walk__positive_closure(self, r):
i, e = self._walk_decorator(r)
if i == e:
self.redge(e, i)
else:
self.edge(e, i)
return (i, e)
def walk__join(self, r):
i, e = self._walk_decorator(r)
n = self.tnode(r.sep)
self.edge(i, n)
self.edge(n, e)
return (i, e)
def walk__group(self, g):
return self._walk_decorator(g)
def walk__choice(self, c):
vopt = [self.walk(o) for o in c.options]
vopt = [o for o in vopt if o is not None]
ni = self.dot()
ne = self.dot()
for i, e in vopt:
self.edge(ni, i)
self.edge(e, ne)
return (ni, ne)
def walk__sequence(self, s):
vseq = [self.walk(x) for x in s.sequence]
vseq = [x for x in vseq if x is not None]
i, _ = vseq[0]
_, e = vseq[-1]
if i != e:
bunch = zip(
[a for _x, a in vseq[:-1]],
[b for b, _y in vseq[1:]],
strict=False,
)
for n, n1 in bunch:
self.edge(n, n1)
return (i, e)
def walk__lookahead(self, la):
_, e = self._walk_decorator(la)
n = self.node('&')
self.edge(n, e)
return (n, e)
def walk__negative_lookahead(self, la):
_, e = self._walk_decorator(la)
n = self.node('!')
self.edge(n, e)
return (n, e)
def walk__rule_include(self, la):
_, e = self._walk_decorator(la)
n = self.node('>')
self.edge(n, e)
return (n, e)
def walk__pattern(self, p):
n = self.tnode(p.pattern)
return (n, n)
def walk__token(self, t):
n = self.tnode(t.token)
return (n, n)
def walk__void(self, v):
n = self.dot()
return (n, n)
def walk__constant(self, t):
n = self.tnode(f'`{t.ast}`')
return (n, n)
def walk__eof(self, v):
# n = self.node('$')
# return (n, n)
return None
@@ -0,0 +1,152 @@
from __future__ import annotations
import re
class ParseException(Exception):
pass
# alias
TatSuException = ParseException
class OptionSucceeded(ParseException):
pass
class GrammarError(ParseException):
pass
class SemanticError(ParseException):
pass
class CodegenError(ParseException):
pass
class MissingSemanticFor(SemanticError):
pass
class ParseError(ParseException):
pass
class FailedSemantics(ParseError):
pass
class FailedKeywordSemantics(FailedSemantics):
pass
class NoParseInfo(ParseException):
pass
class FailedParse(ParseError):
def __init__(self, tokenizer, stack, item):
stack = list(stack) # NOTE: can't pass through multiprocessing if generator
# note: pass all arguments to super() to avoid pickling problems
# https://stackoverflow.com/questions/27993567/
super().__init__(tokenizer, stack, item)
self.tokenizer = tokenizer
self.stack = stack
self.pos = tokenizer.pos
self.item = item
@property
def message(self):
return self.item
def __str__(self):
info = self.tokenizer.line_info(self.pos)
template = '{}({}:{}) {} :\n{}\n{}^\n{}'
text = info.text.rstrip()
leading = re.sub(r'[^\t]', ' ', text)[: info.col]
text = text.expandtabs()
leading = leading.expandtabs()
return template.format(
info.filename,
info.line + 1,
info.col + 1,
self.message.rstrip(),
text,
leading,
'\n'.join(self.stack),
)
class FailedToken(FailedParse):
def __init__(self, tokenizer, stack, token):
super().__init__(tokenizer, stack, token)
self.token = token
@property
def message(self):
return 'expecting {}'.format(repr(self.token).lstrip('u'))
class FailedPattern(FailedParse):
def __init__(self, tokenizer, stack, pattern):
super().__init__(tokenizer, stack, pattern)
self.pattern = pattern
@property
def message(self):
return f'expecting /{self.pattern}/'
class FailedRef(FailedParse):
def __init__(self, tokenizer, stack, name):
super().__init__(tokenizer, stack, name)
self.name = name
@property
def message(self):
return f"could not resolve reference to rule '{self.name}'"
class FailedCut(FailedParse):
def __init__(self, nested):
super().__init__(nested.tokenizer, nested.stack, nested.item)
self.pos = nested.pos
self.nested = nested
@property
def message(self):
return self.nested.message
def __reduce__(self):
return type(self), (self.nested,)
class FailedChoice(FailedParse):
@property
def message(self):
return 'no viable option'
class FailedLookahead(FailedParse):
@property
def message(self):
return 'failed lookahead'
class FailedLeftRecursion(FailedParse):
@property
def message(self):
return 'infinite left recursion'
class FailedExpectingEndOfText(FailedParse):
pass
class FailedKeyword(FailedParse):
pass
@@ -0,0 +1,58 @@
#!/usr/bin/env python3
from __future__ import annotations
import pkgutil
import sys
from pathlib import Path
from tatsu import compile
from .semantics import ANTLRSemantics
def antlr_grammar():
return str(pkgutil.get_data(__name__, 'antlr.ebnf'), 'utf-8')
def translate(
text=None, filename=None, name=None, encoding='utf-8', trace=False,
):
if text is None and filename is None:
raise ValueError('either `text` or `filename` must be provided')
if filename:
filename = Path(filename)
if text is None:
name = name or filename.stem
with filename.open(encoding=encoding) as f:
text = f.read()
name = name or 'Unknown'
semantics = ANTLRSemantics(name)
grammar = compile(antlr_grammar())
model = grammar.parse(
text,
name=name,
filename=filename,
semantics=semantics,
trace=trace,
colorize=True,
)
print(model)
def main():
if len(sys.argv) < 2:
thisprog = Path(sys.argv[0]).name
print(thisprog)
print('Usage:')
print('\t', thisprog, 'FILENAME.g [--trace]')
sys.exit(1)
translate(
filename=sys.argv[1], trace='--trace' in sys.argv or '-t' in sys.argv,
)
if __name__ == '__main__':
main()
@@ -0,0 +1,6 @@
from __future__ import annotations
from . import main
if __name__ == '__main__':
main()
@@ -0,0 +1,197 @@
from __future__ import annotations
import re
from itertools import chain
from tatsu import grammars as model
from tatsu.ast import AST
def camel2py(name):
return re.sub(
r'([a-z0-9])([A-Z])',
lambda m: m.group(1) + '_' + m.group(2).lower(),
name,
)
class ANTLRSemantics:
def __init__(self, name):
self.name = name
self.tokens = {}
self.token_rules = {}
self.synthetic_rules = []
def grammar(self, ast):
return model.Grammar(
self.name,
[
r
for r in chain(ast.rules, self.synthetic_rules)
if r is not None
],
)
def rule(self, ast):
name = camel2py(ast.name)
exp = ast.exp
if name[0].isupper():
name = name.upper()
if isinstance(exp, model.Token):
if name in self.token_rules:
self.token_rules[
name
].exp = exp # it is a model._Decorator
else:
self.token_rules[name] = exp
return None
elif not ast.fragment and not isinstance(exp, model.Sequence):
ref = model.RuleRef(name.lower())
if name in self.token_rules:
self.token_rules[name].exp = ref
else:
self.token_rules[name] = ref
name = name.lower()
return model.Rule(ast, name, exp, ast.params, ast.kwparams)
def alternatives(self, ast):
options = [o for o in ast.options if o is not None]
if len(options) == 1:
return options[0]
else:
options = [model.Option(o) for o in options]
return model.Choice(options)
def elements(self, ast):
elements = [e for e in ast if e is not None]
if not elements:
return model.Void()
elif len(elements) == 1:
return elements[0]
else:
return model.Sequence(AST(sequence=elements))
def predicate_or_action(self, ast):
return None
def named(self, ast):
if ast.force_list:
return model.NamedList(ast)
else:
return model.Named(ast)
def syntactic_predicate(self, ast):
return None
def optional(self, ast):
if isinstance(ast, model.Group | model.Optional | model.Closure):
ast = ast.exp
return model.Optional(ast)
def closure(self, ast):
if isinstance(ast, model.Group | model.Optional):
ast = ast.exp
return model.Closure(ast)
def positive_closure(self, ast):
if isinstance(ast, model.Group):
ast = ast.exp
return model.PositiveClosure(ast)
def negative(self, ast):
neg = model.NegativeLookahead(ast)
any = model.Pattern('.')
return model.Sequence(AST(sequence=[neg, any]))
def subexp(self, ast):
return model.Group(ast)
def regexp(self, ast):
pattern = ''.join(ast)
re.compile(pattern)
return model.Pattern(pattern)
def charset_optional(self, ast):
return f'{ast}?'
def charset_closure(self, ast):
return f'{ast}*'
def charset_positive_closure(self, ast):
return f'{ast}+'
def charset_or(self, ast):
return '[{}]'.format(''.join(ast))
def charset_negative_or(self, ast):
return '[^{}]'.format(''.join(ast))
@staticmethod
def escape(s):
return ''.join('\\' + c if c in '[]().*+{}^$' else c for c in s)
def charset_atom(self, ast):
return ast
def charset_char(self, ast):
return ast
def charset_range(self, ast):
return f'{ast.first}-{ast.last}'
def newranges(self, ast):
pattern = ''.join(ast)
re.compile(pattern)
return model.Pattern(pattern)
def newrange(self, ast):
pattern = '[{}]{}'.format(ast.range, ast.repeat or '')
re.compile(pattern)
return pattern
def negative_newrange(self, ast):
pattern = '[^{}]{}'.format(ast.range, ast.repeat or '')
re.compile(pattern)
return pattern
def rule_ref(self, ast):
assert ast[0].islower()
return model.RuleRef(camel2py(ast))
def any(self, ast):
return model.Pattern(r'\w+|\S+')
def string(self, ast):
text = ast
if isinstance(text, list):
text = ''.join(text)
return model.Token(text)
def eof(self, ast):
return model.EOF()
def token(self, ast):
name = ast.name
if ast.value:
exp = model.Token(ast.value)
self.tokens[name] = exp
else:
exp = model.Fail()
rule = model.Rule(ast, name, exp, [], {})
self.synthetic_rules.append(rule)
return exp
def token_ref(self, ast):
name = camel2py(ast).upper()
value = self.tokens.get(name)
if value and isinstance(value, model.Model):
return value
if name in self.token_rules:
exp = self.token_rules[name]
else:
exp = model.Decorator(model.RuleRef(name))
self.token_rules[name] = exp
return exp
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,269 @@
from __future__ import annotations
import copy
import dataclasses
import re
from collections.abc import Callable, MutableMapping
from itertools import starmap
from typing import Any, NamedTuple
from .ast import AST
from .tokenizing import Tokenizer
from .util.misc import cached_re_compile
from .util.unicode_characters import C_DERIVE
class UndefinedStr(str):
pass
_undefined_str = UndefinedStr('>>undefined<<')
@dataclasses.dataclass
class ParserConfig:
owner: Any = None
name: str | None = 'Test'
filename: str = ''
encoding: str = 'utf-8'
start: str | None = None # FIXME
start_rule: str | None = None # FIXME
rule_name: str | None = None # Backward compatibility
comments_re: re.Pattern | str | None = None
eol_comments_re: re.Pattern | str | None = None
tokenizercls: type[Tokenizer] | None = None # FIXME
semantics: type | None = None
comment_recovery: bool = False
memoize_lookaheads: bool = True
colorize: bool = False
trace: bool = False
trace_filename: bool = False
trace_length: int = 72
trace_separator: str = C_DERIVE
# parser directives
grammar: str | None = None
left_recursion: bool = True
comments: str | None = None
eol_comments: str | None = None
keywords: list[str] | set[str] = dataclasses.field(default_factory=list)
ignorecase: bool | None = False
namechars: str = ''
nameguard: bool | None = None # implied by namechars
whitespace: str | None = _undefined_str
parseinfo: bool = False
def __post_init__(self): # pylint: disable=W0235
if self.ignorecase:
self.keywords = [k.upper() for k in self.keywords]
if self.comments_re or self.eol_comments_re:
raise AttributeError(
"Both `comments_re` and `eol_comments_re` have been removed from parser configurations. " +
"Please use `comments` and/or `eol_comments` instead`.",
)
del self.comments_re
del self.eol_comments_re
if self.comments:
cached_re_compile(self.comments)
if self.eol_comments:
cached_re_compile(self.eol_comments)
if self.whitespace:
cached_re_compile(self.whitespace)
@classmethod
def new(
cls,
config: ParserConfig | None = None,
owner: Any | None = None,
**settings: Any,
) -> ParserConfig:
result = cls(owner=owner)
if config is not None:
result = config.replace_config(config)
return result.replace(**settings)
# NOTE:
# Using functools.cache directly makes objects of this class unhashable
def effective_rule_name(self):
# note: there are legacy reasons for this mess
return self.start_rule or self.rule_name or self.start
def _find_common(self, **settings: Any) -> MutableMapping[str, Any]:
return {
name: value
for name, value in settings.items()
if value is not None and hasattr(self, name)
}
def replace_config(
self, other: ParserConfig | None = None,
) -> ParserConfig:
if other is None:
return self
elif not isinstance(other, ParserConfig):
raise TypeError(f'Unexpected type {type(other).__name__}')
else:
return self.replace(**vars(other))
# non-init fields cannot be used as arguments in `replace`, however
# they are values returned by `vars` and `dataclass.asdict` so they
# must be filtered out.
# If the `ParserConfig` dataclass drops these fields, then this filter can be removed
def _filter_non_init_fields(self, settings: MutableMapping[str, Any]) -> MutableMapping[str, Any]:
for field in [
field.name for field in dataclasses.fields(self) if not field.init
]:
if field in settings:
del settings[field]
return settings
def replace(self, **settings: Any) -> ParserConfig:
overrides = self._filter_non_init_fields(self._find_common(**settings))
result = dataclasses.replace(self, **overrides)
if 'grammar' in overrides:
result.name = result.grammar
return result
def merge(self, **settings: Any) -> ParserConfig:
overrides = self._find_common(**settings)
overrides = {
name: value
for name, value in overrides.items()
if getattr(self, name, None) is None
}
return self.replace(**overrides)
def asdict(self):
# warning: it seems dataclasses.asdict does a deepcopy
# result = dataclasses.asdict(self)
result = copy.copy(vars(self))
result.pop('owner', None)
return result
class PosLine(NamedTuple):
start: int
line: int
length: int
@staticmethod
def build_line_cache(lines):
cache = []
n = 0
i = 0
for n, line in enumerate(lines):
pl = PosLine(i, n, len(line))
for _ in line:
cache.append(pl) # noqa: PERF401
i += len(line)
n += 1
if lines and lines[-1] and lines[-1][-1] in '\r\n':
n += 1
cache.append(PosLine(i, n, 0))
return cache, n
class LineIndexInfo(NamedTuple):
filename: str
line: int
@staticmethod
def block_index(name, n):
return list(
starmap(LineIndexInfo, zip(n * [name], range(n), strict=False)),
)
class LineInfo(NamedTuple):
filename: str
line: int
col: int
start: int
end: int
text: int
class CommentInfo(NamedTuple):
inline: list
eol: list
@staticmethod
def new_comment():
return CommentInfo([], [])
class Alert(NamedTuple):
level: int = 1
message: str = ''
class ParseInfo(NamedTuple):
tokenizer: Any
rule: str
pos: int
endpos: int
line: int
endline: int
alerts: list[Alert] = [] # noqa: RUF012
def text_lines(self):
return self.tokenizer.get_lines(self.line, self.endline)
def line_index(self):
return self.tokenizer.line_index(self.line, self.endline)
@property
def buffer(self):
return self.tokenizer
class MemoKey(NamedTuple):
pos: int
rule: str
state: Any
class RuleInfo(NamedTuple):
name: str
impl: Callable
is_leftrec: bool
is_memoizable: bool
is_name: bool
params: list
kwparams: dict
def __hash__(self):
return hash(self.name)
def __eq__(self, other):
if isinstance(other, RuleInfo):
return self.name == other.name
return False
def __ne__(self, other):
return not self.__eq__(other)
class RuleResult(NamedTuple):
node: Any
newpos: int
newstate: Any
@dataclasses.dataclass(slots=True)
class ParseState:
pos: int = 0
ast: AST = dataclasses.field(default_factory=AST)
cst: Any = None
alerts: list[Alert] = dataclasses.field(default_factory=list)
@@ -0,0 +1,198 @@
# ruff: noqa: PLW2901
from __future__ import annotations
from collections import defaultdict
from collections.abc import Callable
from . import grammars
# Based on https://github.com/ncellar/autumn_v1/
# Returns the correct Rule instance for a RuleRef
def follow(node, rule_dict):
if isinstance(node, grammars.RuleRef):
return rule_dict.get(node.name, node)
else:
return node
class Nullable:
def __init__(self, children, resolved=False, nullable=False):
self.resolved = resolved
self.nullable = nullable
self.children = children
def resolve(self, node, rule_dict):
pass
def resolve_with(self, n):
self.resolved = True
self.nullable = n
self.children = None # No longer needed
all: Callable
any: Callable
of: Callable
no: Callable
yes: Callable
class _All(Nullable):
def resolve(self, node, rule_dict):
unresolved = []
for c in self.children:
c = follow(c, rule_dict)
n = c._nullability
if n.resolved:
if not n.nullable:
# Not nullable if any is not nullable
self.resolve_with(False)
return
else:
unresolved.append(c)
if not unresolved:
# Nullable if all are nullable
self.resolve_with(True)
else:
# Otherwise still unresolved
self.children = unresolved
class _Any(Nullable):
def resolve(self, node, rule_dict):
# Inverse of All
unresolved = []
for c in self.children:
c = follow(c, rule_dict)
n = c._nullability
if n.resolved:
if n.nullable:
self.resolve_with(True)
return
else:
unresolved.append(c)
if not unresolved:
self.resolve_with(False)
else:
self.children = unresolved
class _Single(Nullable):
def resolve(self, node, rule_dict):
n = follow(self.children[0], rule_dict)._nullability
if not n.resolved:
return
self.resolve_with(n.nullable)
Nullable.all = _All # Nullable if all children are nullable
Nullable.any = _Any # Nullable if one child is nullable
Nullable.of = lambda child: _Single(
[child],
) # Nullable if the only child is nullable
Nullable.no = lambda: Nullable(None, True, False) # Not nullable
Nullable.yes = lambda: Nullable(None, True, True) # Nullable
def resolve_nullability(grammar, rule_dict):
dependants = defaultdict(list)
visited = set() # To prevent infinite recursion
def walk(model): # TODO Write a walker for this?
if model in visited:
return
visited.add(model)
for child in model.children_list():
child = follow(child, rule_dict)
walk(child)
resolve(model)
def resolve(model):
nullability = model._nullability
if not nullability.resolved:
nullability.resolve(model, rule_dict)
if nullability.resolved:
for dependant in dependants[model]:
n = dependant._nullability
if not n.resolved:
resolve(
dependant,
) # Resolve nodes that depend on this one
else:
for n in nullability.children:
dependants[n].append(model)
walk(grammar)
# This breaks left recursive cycles by tagging
# left recursive rules
def find_left_recursion(grammar):
rule_dict = {
rule.name: rule for rule in grammar.rules
} # Required to resolve rule references
# First we need to resolve nullable rules
resolve_nullability(grammar, rule_dict)
# Traversable state
FIRST = 0
CUTOFF = 1
VISITED = 2
state = defaultdict(lambda: FIRST)
stack_depth = [0] # nonlocal workaround 2.7
stack_positions = {}
lr_stack_positions = [-1]
def walk(model):
if state[model] == FIRST:
state[model] = CUTOFF
else:
return
# beforeNode
leftrec = isinstance(model, grammars.Rule) and model.is_leftrec
if leftrec:
lr_stack_positions.append(stack_depth[0])
stack_positions[model] = stack_depth[0]
stack_depth[0] += 1
for child in model.at_same_pos(rule_dict):
child = follow(child, rule_dict)
walk(child)
# afterEdge
if (
state[child] == CUTOFF
and stack_positions[child] > lr_stack_positions[-1]
):
# turn off memoization for all rules that were involved in this cycle
for rule in stack_positions:
if isinstance(rule, grammars.Rule):
rule.is_memoizable = False
child.is_leftrec = True
# afterNode
if leftrec:
lr_stack_positions.pop()
del stack_positions[model]
stack_depth[0] -= 1
state[model] = VISITED
for rule in grammar.children_list():
walk(rule)
# print()
# for rule in grammar.children_list():
# print(rule)
# if rule.is_leftrec: print("-> Leftrec")
# if rule.is_nullable(): print("-> Nullable")
@@ -0,0 +1,65 @@
import io
from contextlib import contextmanager
from ..util import trim
class IndentPrintMixin:
def __init__(self, default_indent: int = 4):
self.default_indent = default_indent
self.indent_stack: list[int] = [0]
self.output_stream = io.StringIO()
def printed_text(self):
return self.output_stream.getvalue()
def print(self, *args, **kwargs):
args = [trim(str(arg)) for arg in args if arg is not None]
lines = self.as_printed_lines(*args, **kwargs)
self._do_print_lines(lines, **kwargs)
def as_printed(self, *args, **kwargs):
lines = self.as_printed_lines(*args, **kwargs)
return '\n'.join(lines)
def as_printed_lines(self, *args, **kwargs):
text = self.io_print(*args, **kwargs)
return self.indented_lines(text)
@contextmanager
def indent(self, amount: int | None = None):
assert amount is None or amount >= 0
if amount is None:
amount = self.default_indent
self.indent_stack.append(amount + self.indent_stack[-1])
try:
yield
finally:
self.indent_stack.pop()
@property
def current_indentation(self):
return ' ' * self.indent_stack[-1]
@staticmethod
def io_print(*args, **kwargs):
kwargs.pop('file', None) # do not allow redirection of output
with io.StringIO() as output:
print(*args, file=output, **kwargs)
return output.getvalue()
def _do_print_lines(self, lines: list[str] | None = None, **kwargs):
if not lines:
print(file=self.output_stream, **kwargs)
return
for line in lines:
print(line, file=self.output_stream, **kwargs)
def indented_lines(self, text):
text = trim(text)
return [
(self.current_indentation + line).rstrip()
for line in text.splitlines(keepends=False)
]
@@ -0,0 +1,7 @@
from __future__ import annotations
from .ast import AST # noqa: F401
from .objectmodel import Node # noqa: F401
from .objectmodel import Node as ParseModel # noqa: F401
from .semantics import ModelBuilderSemantics # noqa: F401
from .walkers import DepthFirstWalker, NodeWalker # noqa: F401 # noqa: F401
@@ -0,0 +1,8 @@
from ..objectmodel import ParseModel
from .python import PythonCodeGenerator
def codegen(model: ParseModel, parser_name: str = '') -> str:
generator = PythonCodeGenerator(parser_name=parser_name)
generator.walk(model)
return generator.printed_text()
@@ -0,0 +1,149 @@
import builtins
from collections import namedtuple
from .. import grammars, objectmodel
from ..mixins.indent import IndentPrintMixin
from ..util import compress_seq, safe_name
from ..util.misc import topsort
HEADER = """\
#!/usr/bin/env python3
# WARNING: CAVEAT UTILITOR
#
# This file was automatically generated by TatSu.
#
# https://pypi.python.org/pypi/tatsu/
#
# Any changes you make to it will be overwritten the next time
# the file is generated.
from __future__ import annotations
from typing import Any
from dataclasses import dataclass
from tatsu.semantics import ModelBuilderSemantics
{base_type_import}
class {name}ModelBuilderSemantics(ModelBuilderSemantics):
def __init__(self, context=None, types=None):
types = [
t for t in globals().values()
if type(t) is type and issubclass(t, ModelBase)
] + (types or [])
super().__init__(context=context, types=types)
"""
BaseClassSpec = namedtuple('BaseClassSpec', ['class_name', 'base'])
def modelgen(model: grammars.Grammar, name: str = '', base_type: type | None = objectmodel.Node) -> str:
base_type = base_type or objectmodel.Node
generator = PythonModelGenerator(name=name, base_type=base_type)
return generator.generate_model(model)
class PythonModelGenerator(IndentPrintMixin):
def __init__(self, name: str = '', base_type: type = objectmodel.Node):
super().__init__()
self.base_type = base_type
self.name = name or None
def generate_model(self, grammar: grammars.Grammar):
base_type = self.base_type
base_type_name = base_type.__name__.split('.')[-1]
base_type_import = f"from {base_type.__module__} import {base_type_name}"
self.name = self.name or grammar.name
self.print(
HEADER.format(
name=self.name,
base_type=self.base_type.__name__,
base_type_import=base_type_import,
),
)
rule_index = {rule.name: rule for rule in grammar.rules}
rule_specs = {
rule.name: self._base_class_specs(rule)
for rule in grammar.rules
}
rule_specs = {name: specs for name, specs in rule_specs.items() if specs}
specs_by_name = {
s.class_name: s.base
for specs in rule_specs.values()
for s in specs
}
base = self._model_base_name()
specs_by_name[base] = base_type_name
all_specs = {
(s.class_name, s.base)
for specs in rule_specs.values()
for s in specs
}
model_names = topsort(reversed(specs_by_name), all_specs)
model_to_rule = {
rule_specs[name][0].class_name: rule
for name, rule in rule_index.items()
if name in rule_specs
}
for model_name in model_names:
if model_name in vars(builtins):
continue
if rule := model_to_rule.get(model_name):
self._gen_rule_class(rule, rule_specs[rule.name])
else:
self._gen_base_class(model_name, specs_by_name.get(model_name))
return self.printed_text()
@staticmethod
def _model_base_name():
return 'ModelBase'
def _gen_base_class(self, class_name: str, base: str | None):
self.print()
self.print()
self.print('@dataclass(eq=False)')
if base:
self.print(f'class {class_name}({base}):')
else:
# FIXME: this cannot happen as base_type is the final base
self.print(f'class {class_name}:')
with self.indent():
self.print('pass')
def _gen_rule_class(self, rule: grammars.Rule, specs: list[BaseClassSpec]):
if not specs:
return
spec = specs[0]
arguments = sorted({safe_name(d) for d, _ in compress_seq(rule.defines())})
self.print()
self.print()
self.print('@dataclass(eq=False)')
self.print(f'class {spec.class_name}({spec.base}):')
with self.indent():
if not arguments:
self.print('pass')
for arg in arguments:
self.print(f'{arg}: Any = None')
def _base_class_specs(self, rule: grammars.Rule) -> list[BaseClassSpec]:
if not rule.params or not isinstance(rule.params[0], str):
return []
spec = rule.params[0].split('::')
base = [self._model_base_name()]
class_names = [safe_name(n) for n in spec] + base
return [
BaseClassSpec(class_name, class_names[i + 1])
for i, class_name in enumerate(class_names[:-1])
]
@@ -0,0 +1,392 @@
from __future__ import annotations
import itertools
import textwrap
from collections.abc import Iterator
from typing import Any
from .. import grammars
from ..collections import OrderedSet as oset
from ..exceptions import CodegenError
from ..infos import UndefinedStr
from ..mixins.indent import IndentPrintMixin
from ..util import compress_seq, safe_name
from ..walkers import NodeWalker
HEADER = """\
#!/usr/bin/env python3
# WARNING: CAVEAT UTILITOR
#
# This file was automatically generated by TatSu.
#
# https://pypi.python.org/pypi/tatsu/
#
# Any changes you make to it will be overwritten the next time
# the file is generated.
# ruff: noqa: C405, COM812, I001, F401, PLR1702, PLC2801, SIM117
import sys
from pathlib import Path
from tatsu.buffering import Buffer
from tatsu.parsing import Parser
from tatsu.parsing import tatsumasu
from tatsu.parsing import leftrec, nomemo, isname
from tatsu.infos import ParserConfig
from tatsu.util import re, generic_main
"""
FOOTER = """\
def main(filename, **kwargs):
if not filename or filename == '-':
text = sys.stdin.read()
else:
text = Path(filename).read_text()
parser = {name}Parser()
return parser.parse(
text,
filename=filename,
**kwargs,
)
if __name__ == '__main__':
import json
from tatsu.util import asjson
ast = generic_main(main, {name}Parser, name='{name}')
data = asjson(ast)
print(json.dumps(data, indent=2))
"""
class PythonCodeGenerator(IndentPrintMixin, NodeWalker):
_counter: Iterator[int] = itertools.count()
def __init__(self, parser_name: str = ''):
super().__init__()
self.parser_name = parser_name
@classmethod
def _next_n(cls):
return next(cls._counter)
@classmethod
def _reset_counter(cls):
cls._counter = itertools.count()
def walk_default(self, node: Any):
return node
def walk_Grammar(self, grammar: grammars.Grammar):
parser_name = self.parser_name or grammar.name
self.print(HEADER)
self.print()
self.print()
self._gen_keywords(grammar)
self._gen_buffering(grammar, parser_name)
self._gen_parsing(grammar, parser_name)
self.print(FOOTER.format(name=parser_name))
def walk_Rule(self, rule: grammars.Rule):
def param_repr(p):
if isinstance(p, int | float):
return str(p)
else:
return repr(p.split('::')[0])
self._reset_counter()
params = kwparams = ''
if rule.params:
params = ', '.join(
param_repr(self.walk(p)) for p in rule.params
)
if rule.kwparams:
kwparams = ', '.join(
f'{k}={param_repr(self.walk(v))}'
for k, v in rule.kwparams.items()
)
if params and kwparams:
params = params + ', ' + kwparams
elif kwparams:
params = kwparams
leftrec = '\n@leftrec' if rule.is_leftrec else ''
nomemo = (
'\n@nomemo'
if not rule.is_memoizable and not leftrec
else ''
)
isname = '\n@isname' if rule.is_name else ''
self.print(
f"""
@tatsumasu({params})\
{leftrec}\
{nomemo}\
{isname}\
\ndef _{rule.name}_(self):
""",
)
with self.indent():
self.print(self.walk(rule.exp))
def walk_BasedRule(self, rule: grammars.BasedRule):
# FIXME: the following override is to not alter the previous codegen
rule.exp = rule.rhs
self.walk_Rule(rule)
def walk_RuleRef(self, ref: grammars.RuleRef):
self.print(f'self._{ref.name}_()')
def walk_RuleInclude(self, include: grammars.RuleInclude):
self.walk(include.rule.exp)
def walk_Void(self, void: grammars.Void):
self.print('self._void()')
def walk_Any(self, any: grammars.Any):
self.print('self._any()')
def walk_Fail(self, fail: grammars.Fail):
self.print('self._fail()')
def walk_Cut(self, cut: grammars.Cut):
self.print('self._cut()')
def walk_Comment(self, comment: grammars.Comment):
lines = '\n'.join(f'# {c!s}' for c in comment.comment.splitlines())
self.print(f'\n{lines}\n')
def walk_EOLComment(self, comment: grammars.EOLComment):
self.walk_Comment(comment)
def walk_EOF(self, eof: grammars.EOF):
self.print('self._check_eof()')
def walk_Group(self, group: grammars.Group):
self.print('with self._group():')
with self.indent():
self.walk(group.exp)
def walk_Token(self, token: grammars.Token):
self.print(f'self._token({token.token!r})')
def walk_Constant(self, constant: grammars.Constant):
self.print(f'self._constant({constant.literal!r})')
def walk_Alert(self, alert: grammars.Alert):
self.print(f'self._alert({alert.literal!r}, {alert.level})')
def walk_Pattern(self, pattern: grammars.Pattern):
self.print(f'self._pattern({pattern.pattern!r})')
def walk_Lookahead(self, lookahead: grammars.Lookahead):
self.print('with self._if():')
with self.indent():
self.walk(lookahead.exp)
def walk_NegativeLookahead(self, lookahead: grammars.NegativeLookahead):
self.print('with self._ifnot():')
with self.indent():
self.walk(lookahead.exp)
def walk_Sequence(self, seq: grammars.Sequence):
self.walk(seq.sequence)
self._gen_defines_declaration(seq)
def walk_Choice(self, choice: grammars.Choice):
if len(choice.options) == 1:
self.walk(choice.options[0])
return
firstset = choice.lookahead_str()
if firstset:
msglines = textwrap.wrap(firstset, width=40)
error = ['expecting one of: ', *msglines]
else:
error = ['no available options']
errors = '\n'.join(repr(e) for e in error)
self.print('with self._choice():')
with self.indent():
self.walk(choice.options)
self.print('self._error(')
with self.indent():
self.print(errors)
self.print(')')
def walk_Option(self, option: grammars.Option):
self.print('with self._option():')
with self.indent():
self.walk(option.exp)
def walk_Optional(self, optional: grammars.Optional):
self.print('with self._optional():')
with self.indent():
self.walk(optional.exp)
def walk_EmptyClosure(self, closure: grammars.EmptyClosure):
self.print('self._empty_closure()')
def walk_Closure(self, closure: grammars.Closure):
n = self._gen_block(closure.exp)
self.print(f'self._closure(block{n})')
def walk_PositiveClosure(self, closure: grammars.PositiveClosure):
n = self._gen_block(closure.exp)
self.print(f'self._positive_closure(block{n})')
def walk_Join(self, join: grammars.Join):
m = self._gen_block(join.sep, name='sep')
n = self._gen_block(join.exp)
self.print(f'self._join(block{n}, sep{m})')
def walk_PositiveJoin(self, join: grammars.PositiveJoin):
m = self._gen_block(join.sep, name='sep')
n = self._gen_block(join.exp)
self.print(f'self._positive_join(block{n}, sep{m})')
def walk_LeftJoin(self, join: grammars.LeftJoin):
m = self._gen_block(join.sep, name='sep')
n = self._gen_block(join.exp)
self.print(f'self._left_join(block{n}, sep{m})')
def walk_RightJoin(self, join: grammars.RightJoin):
m = self._gen_block(join.sep, name='sep')
n = self._gen_block(join.exp)
self.print(f'self._right_join(block{n}, sep{m})')
def walk_Gather(self, gather: grammars.Gather):
m = self._gen_block(gather.sep, name='sep')
n = self._gen_block(gather.exp)
self.print(f'self._gather(block{n}, sep{m})')
def walk_PositiveGather(self, gather: grammars.PositiveGather):
m = self._gen_block(gather.sep, name='sep')
n = self._gen_block(gather.exp)
self.print(f'self._positive_gather(block{n}, sep{m})')
def walk_SkipTo(self, skipto: grammars.SkipTo):
n = self._gen_block(skipto.exp)
self.print(f'self._skip_to(block{n})')
def walk_Named(self, named: grammars.Named):
self.walk(named.exp)
self.print(f"self.name_last_node('{named.name}')")
def walk_NamedList(self, named: grammars.Named):
self.walk(named.exp)
self.print(f"self.add_last_node_to_name('{named.name}')")
def walk_Override(self, override: grammars.Override):
self.walk_Named(override)
def walk_OverrideList(self, override: grammars.OverrideList):
self.walk_NamedList(override)
def walk_Special(self, special: grammars.Special):
pass
def _gen_keywords(self, grammar: grammars.Grammar):
keywords = [str(k) for k in grammar.keywords if k is not None]
if not keywords:
self.print('KEYWORDS: set[str] = set()')
else:
keywords_str = '\n'.join(f' {k!r},' for k in keywords)
keywords_str = f'{{\n{keywords_str}\n}}'
self.print(f'KEYWORDS: set[str] = {keywords_str}')
self.print()
self.print()
def _gen_init(self, grammar: grammars.Grammar):
start = grammar.config.start or grammar.rules[0].name
whitespace = grammar.config.whitespace
if isinstance(whitespace, UndefinedStr): # the default value
whitespace = None
whitespace = repr(whitespace)
self.print(
f'''
config = ParserConfig.new(
config,
owner=self,
whitespace={whitespace},
nameguard={grammar.config.nameguard},
ignorecase={grammar.config.ignorecase},
namechars={grammar.config.namechars!r},
parseinfo={grammar.config.parseinfo},
comments={grammar.config.comments!r},
eol_comments={grammar.config.eol_comments!r},
keywords=KEYWORDS,
start={start!r},
)
config = config.replace(**settings)
''',
)
self.print()
def _gen_buffering(self, grammar: grammars.Grammar, parser_name: str):
self.print(f'class {parser_name}Buffer(Buffer):')
with self.indent():
self.print('def __init__(self, text, /, config: ParserConfig | None = None, **settings):')
with self.indent():
self._gen_init(grammar)
self.print('super().__init__(text, config=config)')
self.print()
def _gen_parsing(self, grammar: grammars.Grammar, parser_name: str):
self.print()
self.print(f'class {parser_name}Parser(Parser):')
with self.indent():
self.print('def __init__(self, /, config: ParserConfig | None = None, **settings):')
with self.indent():
self._gen_init(grammar)
self.print('super().__init__(config=config)')
self.print()
self.walk(grammar.rules)
self.print()
def _gen_defines_declaration(self, node: grammars.Model):
defines = compress_seq(node.defines())
ldefs = oset(safe_name(d) for d, value in defines if value)
sdefs = oset(
safe_name(d)
for d, value in defines
if not value and d not in ldefs
)
if not (sdefs or ldefs):
return
sdefs_str = ', '.join(sorted(repr(d) for d in sdefs))
ldefs_str = ', '.join(sorted(repr(d) for d in ldefs))
if not ldefs:
self.print(f'self._define([{sdefs_str}], [{ldefs_str}])')
else:
self.print('self._define(')
with self.indent():
self.print(f'[{sdefs_str}],')
self.print(f'[{ldefs_str}],')
self.print(')')
def _gen_block(self, exp: grammars.Model, name='block'):
if () in exp.lookahead():
raise CodegenError(f'{exp} may repeat empty sequence')
n = self._next_n()
self.print()
self.print(f'def {name}{n}():')
with self.indent():
self.walk(exp)
return n
@@ -0,0 +1,203 @@
from __future__ import annotations
import weakref
from collections.abc import Mapping
from dataclasses import dataclass
from typing import Any
from .ast import AST
from .infos import CommentInfo, ParseInfo
from .util import AsJSONMixin, asjson, asjsons
BASE_CLASS_TOKEN = '::' # noqa: S105
@dataclass(eq=False)
class Node(AsJSONMixin):
_parent: Node | None = None
_children: list[Node] | None = None
ast: AST | None = None
ctx: Any = None
parseinfo: ParseInfo | None = None
def __init__(self, ast=None, **attributes):
super().__init__()
if isinstance(ast, dict):
ast = AST(ast)
self.ast = ast
for name, value in attributes.items():
setattr(self, name, value)
self.__post_init__()
def __post_init__(self):
ast = self.ast
if not self.parseinfo and isinstance(ast, AST):
self.parseinfo = ast.parseinfo
if not isinstance(ast, Mapping):
return
for name in set(ast) - {'parseinfo'}:
try:
setattr(self, name, ast[name])
except AttributeError as e:
raise AttributeError(f"'{name}' is a reserved name") from e
self._children = self.children_list()
if self.parseinfo is None:
del self.parseinfo
@property
def parent(self):
return self._parent
@property
def line(self):
if self.parseinfo:
return self.parseinfo.line
return None
@property
def endline(self):
if self.parseinfo:
return self.parseinfo.endline
return None
def text_lines(self):
if self.parseinfo:
return self.parseinfo.text_lines()
return None
def line_index(self):
return self.parseinfo.line_index()
@property
def col(self):
return self.line_info.col if self.line_info else None
@property
def context(self):
return self.ctx
@property
def line_info(self):
if self.parseinfo:
return self.parseinfo.tokenizer.line_info(self.parseinfo.pos)
return None
@property
def text(self):
if not self.parseinfo:
return ''
text = self.parseinfo.tokenizer.text
return text[self.parseinfo.pos: self.parseinfo.endpos]
@property
def comments(self):
if self.parseinfo:
return self.parseinfo.tokenizer.comments(self.parseinfo.pos)
return CommentInfo([], [])
@property
def _deref(self):
# use this to get the actual object over weakref instances
return self
def _find_children(self):
def with_parent(node):
node._parent = weakref.proxy(self)
return node
def children_of(child):
if isinstance(child, weakref.ReferenceType | weakref.ProxyType):
return
elif isinstance(child, Node):
yield with_parent(child)
elif isinstance(child, Mapping):
for name, value in child.items():
if name.startswith('_'):
continue
yield from children_of(value)
elif isinstance(child, list | tuple):
yield from (
with_parent(c) for c in child if isinstance(c, Node)
)
children = list(self._pubdict().items())
if not children:
yield from children_of(self.ast)
else:
for name, child in children:
if name.startswith('_'):
continue
yield from children_of(child)
def children_list(self):
if self._children is not None:
return self._children
return list(self._find_children())
def children_set(self):
return set(self.children_list())
def children(self):
return self.children_list()
def asjson(self):
return asjson(self)
def _pubdict(self):
return {
name: value
for name, value in super()._pubdict().items()
if name not in {'ast', 'ctx', 'parent', 'parseinfo'}
}
def __str__(self):
return asjsons(self)
def __hash__(self):
if self.ast is not None:
if isinstance(self.ast, list):
return hash(tuple(self.ast))
elif isinstance(self.ast, dict):
return hash(AST(self.ast))
else:
return hash(self.ast)
else:
return id(self)
def __eq__(self, other):
if id(self) == id(other):
return True
elif self.ast is None:
return False
elif not getattr(other, 'ast', None):
return False
else:
return self.ast == other.ast
def _nonrefdict(self):
return {
name: value
for name, value in vars(self).items()
if (
name not in {'_parent', '_children'}
and type(value)
not in {weakref.ReferenceType, weakref.ProxyType}
)
}
# NOTE: pickling is important for parallel parsing
def __getstate__(self):
return self._nonrefdict()
def __setstate__(self, state):
self.__dict__.update(state)
self.children_list()
ParseModel = Node
@@ -0,0 +1,86 @@
import re
from typing import Any
from .bootstrap import EBNFBootstrapParser
from .buffering import Buffer
from .grammars import PRAGMA_RE
from .infos import ParserConfig
from .parser_semantics import EBNFGrammarSemantics
from .semantics import ASTSemantics
class EBNFBuffer(Buffer):
def __init__(
self,
text,
/,
filename=None,
config: ParserConfig | None = None,
**settings: Any,
):
config = ParserConfig.new(
config=config, owner=self, filename=filename, **settings,
)
super().__init__(text, config=config)
def process_block(self, name, lines, index, **kwargs):
i = 0
while i < len(lines):
line = lines[i]
if re.match(PRAGMA_RE, line):
directive, arg = line.split('#', 1)[1], ''
if '::' in directive:
directive, arg = directive.split('::', 1)
directive, arg = directive.strip(), arg.strip()
i = self.pragma(name, directive, arg, lines, index, i)
else:
i += 1
return lines, index
def pragma(self, source, name, arg, lines, index, i):
# we only recognize the 'include' pragama
if name == 'include':
filename = arg.strip('\'"')
return self.include_file(source, filename, lines, index, i, i + 1)
else:
return i + 1 # will be treated as a directive by the parser
class EBNFParser(EBNFBootstrapParser):
def __init__(
self,
name: str | None = None,
config: ParserConfig | None = None,
semantics=None,
**settings: Any,
):
if semantics is None:
semantics = ASTSemantics()
config = ParserConfig.new(
config=config,
name=name,
semantics=semantics,
tokenizercls=EBNFBuffer,
**settings,
)
super().__init__(config)
class GrammarGenerator(EBNFBootstrapParser):
def __init__(
self,
name: str | None = None,
config: ParserConfig | None = None,
semantics=None,
**settings: Any,
):
if semantics is None:
semantics = EBNFGrammarSemantics(name)
config = ParserConfig.new(
config=config,
name=name,
semantics=semantics,
tokenizercls=EBNFBuffer,
**settings,
)
super().__init__(config)
@@ -0,0 +1,148 @@
from __future__ import annotations
from collections.abc import Sequence
from . import grammars
from .exceptions import FailedSemantics
from .semantics import ModelBuilderSemantics
from .util import eval_escapes, flatten, re, warning
class EBNFGrammarSemantics(ModelBuilderSemantics):
def __init__(self, grammar_name):
super().__init__(
base_type=grammars.Model, types=grammars.Model.classes(),
)
self.grammar_name = grammar_name
self.rules = {}
def token(self, ast, *args):
token = ast
if not token:
raise FailedSemantics('empty token')
return grammars.Token(token)
def pattern(self, ast, *args):
return grammars.Pattern(ast)
def regexes(self, ast, *args):
pattern = ''.join(ast)
try:
re.compile(pattern)
except (TypeError, re.error) as e:
raise FailedSemantics('regexp error: ' + str(e)) from e
return ast
def regex(self, ast, *args):
pattern = ast
try:
re.compile(pattern)
except (TypeError, re.error) as e:
raise FailedSemantics('regexp error: ' + str(e)) from e
return pattern
def string(self, ast):
return eval_escapes(ast)
def hex(self, ast):
return int(ast, 16)
def float(self, ast):
return float(ast)
def int(self, ast):
return int(ast)
def null(self, ast):
return None
def cut_deprecated(self, ast, *args):
warning(
'The use of >> for cut is deprecated. Use the ~ symbol instead.',
)
return grammars.Cut()
def override_single_deprecated(self, ast, *args):
warning('The use of @ for override is deprecated. Use @: instead')
return grammars.Override(ast)
def sequence(self, ast, *args):
seq = ast.sequence
assert isinstance(seq, Sequence), str(seq)
if len(seq) == 1:
return seq[0]
return grammars.Sequence(ast)
def choice(self, ast, *args):
if len(ast) == 1:
return ast[0]
return grammars.Choice(ast)
def new_name(self, name):
if name in self.rules:
raise FailedSemantics(f'rule "{name!s}" already defined')
return name
def known_name(self, name):
if name not in self.rules:
raise FailedSemantics(f'rule "{name!s}" not yet defined')
return name
def boolean(self, ast):
return str(ast).lower() in {'true', 'yes', 'ok', '1'}
def rule(self, ast, *args):
decorators = ast.decorators
name = ast.name
exp = ast.exp
base = ast.base
params = ast.params
kwparams = dict(ast.kwparams) if ast.kwparams else None
if 'override' not in decorators and name in self.rules:
self.new_name(name)
elif 'override' in decorators:
self.known_name(name)
if not base:
rule = grammars.Rule(
ast, name, exp, params, kwparams, decorators=decorators,
)
else:
self.known_name(base)
base_rule = self.rules[base]
rule = grammars.BasedRule(
ast,
name,
exp,
base_rule,
params,
kwparams,
decorators=decorators,
)
self.rules[name] = rule
return rule
def rule_include(self, ast, *args):
name = str(ast)
self.known_name(name)
rule = self.rules[name]
return grammars.RuleInclude(rule)
def grammar(self, ast, *args):
directives = {d.name: d.value for d in flatten(ast.directives)}
keywords = list(flatten(ast.keywords)) or []
if directives.get('whitespace') in {'None', 'False'}:
# NOTE: use '' because None will _not_ override defaults in configuration
directives['whitespace'] = ''
name = self.grammar_name or directives.get('grammar')
return grammars.Grammar(
name,
list(self.rules.values()),
directives=directives,
keywords=keywords,
)
@@ -0,0 +1,38 @@
from __future__ import annotations
import inspect
from .contexts import ( # noqa: F401
ParseContext,
isname,
leftrec,
nomemo,
tatsumasu,
)
from .exceptions import FailedRef
class Parser(ParseContext):
def _find_rule(self, name):
rule = getattr(self, '_' + name + '_', None)
if isinstance(rule, type(self._find_rule)):
return rule
rule = getattr(self, name, None)
if isinstance(rule, type(self._find_rule)):
return rule
self._error(name, exclass=FailedRef)
return None
@classmethod
def rule_list(cls):
methods = inspect.getmembers(cls, predicate=inspect.isroutine)
result = []
for m in methods:
name = m[0]
if len(name) < 3:
continue
if name.startswith('__') or name.endswith('__'):
continue
if name.startswith('_') and name.endswith('_'):
result.append(name[1:-1])
return result
@@ -0,0 +1,141 @@
"""
The Renderer class provides the infrastructure for generating template-based
code. It's used by the .grammars module for parser generation.
"""
from __future__ import annotations
import itertools
import string
from .util import indent, isiter, trim
def render(item, join='', **fields):
"""Render the given item"""
if item is None:
return ''
elif isinstance(item, str):
return item
elif isinstance(item, Renderer):
return item.render(join=join, **fields)
elif isiter(item):
return join.join(
render(e, **fields) for e in iter(item) if e is not None
)
elif isinstance(item, int | float):
return item
else:
return str(item)
class RenderingFormatter(string.Formatter):
def render(self, item, join='', **fields):
return render(item, join=join, **fields)
def format_field(self, value, format_spec):
if ':' not in format_spec:
return super().format_field(self.render(value), format_spec)
ind, sep, fmt = format_spec.split(':')
if sep == '\\n':
sep = '\n'
if not ind:
ind = 0
mult = 0
elif '*' in ind:
ind, mult = ind.split('*')
else:
mult = 4
ind = int(ind)
mult = int(mult)
if not fmt:
fmt = '%s'
if isiter(value):
return indent(
sep.join(fmt % self.render(v) for v in value), ind, mult,
)
else:
return indent(fmt % self.render(value), ind, mult)
class Renderer:
"""Renders the fileds in the current object using a template
provided statically, on the constructor, or as a parameter
to render().
Fields with a leading underscore are not made available to
the template. Additional fields may be made available by
overriding render_fields().
"""
template = '{__class__}'
_counter = itertools.count()
_formatter = RenderingFormatter()
def __init__(self, template=None):
if template is not None:
self.template = template
@classmethod
def counter(cls):
return next(cls._counter)
@classmethod
def reset_counter(cls):
Renderer._counter = itertools.count()
@property
def formatter(self):
return self._formatter
@formatter.setter
def formatter(self, value):
self._formatter = value
def rend(self, item, join='', **fields):
"""A shortcut for self._formatter.render()"""
return self._formatter.render(item, join=join, **fields)
def indent(self, item, ind=1, multiplier=4):
return indent(self.rend(item), indent=ind, multiplier=multiplier)
def trim(self, item, tabwidth=4):
return trim(self.rend(item), tabwidth=tabwidth)
def render_fields(self, fields):
"""Pre-render fields before rendering the template."""
return
def render(self, **fields):
template = fields.pop('template', None)
fields.update(__class__=self.__class__.__name__)
fields.update(
{k: v for k, v in vars(self).items() if not k.startswith('_')},
)
override = self.render_fields(
fields,
) # pylint: disable=assignment-from-none
if override is not None:
template = override
elif template is None:
template = self.template
try:
return self._formatter.format(trim(template), **fields)
except KeyError as e:
# find the missing key
keys = (p[1] for p in self._formatter.parse(template))
for key in keys:
if key and key not in fields:
raise KeyError(key, type(self)) from e
raise
def __str__(self):
return self.render()
def __repr__(self):
return str(self)
@@ -0,0 +1,95 @@
from __future__ import annotations
import builtins
from .exceptions import SemanticError
from .objectmodel import BASE_CLASS_TOKEN, Node
from .synth import synthesize
from .util import simplify_list
class ASTSemantics:
def group(self, ast, *args):
return simplify_list(ast)
def element(self, ast, *args):
return simplify_list(ast)
def sequence(self, ast, *args):
return simplify_list(ast)
def choice(self, ast, *args):
if len(ast) == 1:
return simplify_list(ast[0])
return ast
class ModelBuilderSemantics:
"""Intended as a semantic action for parsing, a ModelBuilderSemantics creates
nodes using the class name given as first parameter to a grammar
rule, and synthesizes the class/type if it's not known.
"""
def __init__(self, context=None, base_type=Node, types=None):
self.ctx = context
self.base_type = base_type
self.constructors = {}
for t in types or ():
self._register_constructor(t)
def _register_constructor(self, constructor):
self.constructors[constructor.__name__] = constructor
return constructor
def _find_existing_constructor(self, typename):
constructor = builtins
for name in typename.split('.'):
try:
context = vars(constructor)
except Exception as e:
raise SemanticError(
f'Could not find constructor for {typename} ({type(constructor).__name__}): {e!s}',
) from e
if name in context:
constructor = context[name]
else:
constructor = None
break
return constructor
def _get_constructor(self, typename, base):
typename = str(typename)
if typename in self.constructors:
return self.constructors[typename]
constructor = self._find_existing_constructor(typename)
if not constructor:
constructor = synthesize(typename, base)
return self._register_constructor(constructor)
def _default(self, ast, *args, **kwargs):
if not args:
return ast
typespec = args[0].split(BASE_CLASS_TOKEN)
typename = typespec[0]
bases = typespec[-1:0:-1]
base = self.base_type
for base_ in bases:
base = self._get_constructor(base_, base)
constructor = self._get_constructor(typename, base)
try:
if isinstance(constructor, type) and issubclass(constructor, Node):
return constructor(ast=ast, ctx=self.ctx, **kwargs)
else:
return constructor(ast, *args[1:], **kwargs)
except Exception as e:
raise SemanticError(
f'Could not call constructor for {typename}: {e!s}',
) from e
@@ -0,0 +1,320 @@
from __future__ import annotations
import functools
import operator
import weakref
from collections import defaultdict
from copy import copy
from .exceptions import ParseException
from .infos import LineIndexInfo
from .util import asjson, join_lists
DEFAULT_SEPARATOR = '.'
def join_symtables(tables):
def join_namespaces(base, target):
base = copy(base)
for symbol in target.symbols:
name = symbol.name
if name not in base or base.duplicates:
base.insert(symbol)
else:
join_namespaces(base[name], symbol)
return base
if not tables:
return {}
result = copy(tables[0])
for table in tables[1:]:
result = join_namespaces(table, result)
return result
class SymbolTableError(ParseException):
pass
class Namespace:
def __init__(
self, ignorecase=False, duplicates=False, separator=DEFAULT_SEPARATOR,
):
super().__init__()
self.ignorecase = ignorecase
self.duplicates = duplicates
self.separator = separator
self._entries = defaultdict(list)
@property
def entries(self):
return self._entries
@property
def symbols(self):
return join_lists(self._entries.values())
@property
def names(self):
return list(self._entries.keys())
def all_names(self):
result = []
for name, symbols in self.entries.items():
result.append(name)
for symbol in symbols:
result.extend(symbol.all_names())
return result
def __contains__(self, name):
if self.ignorecase:
name = name.upper()
return name in self.entries
def __getitem__(self, name):
if self.ignorecase:
name = name.upper()
if self.duplicates:
return self.entries.get(name)
elif name in self.entries:
return self.entries[name][0]
else:
raise KeyError(name)
def get(self, name, default=None):
return self.entries.get(name, default)
def insert(self, symbol):
assert isinstance(
symbol.name, str,
), f'"{symbol.name!s}" is not a valid symbol name'
name = symbol.name
if self.ignorecase:
name = name.upper()
if name in self._entries and not self.duplicates:
raise SymbolTableError(
f'Symbol "{symbol.name!s}" already in namespace',
)
self._entries[name].append(symbol)
def lookup_all(self, qualname, drill=True):
if self.ignorecase:
qualname = qualname.upper()
return self._lookup_drilldown(
qualname.split(self.separator), drill=drill,
)
def lookup(self, qualname, drill=True):
if self.ignorecase:
qualname = qualname.upper()
result = self._lookup_drilldown(
qualname.split(self.separator), drill=drill, max=1,
)
return result[0] if result else None
def _lookup_drilldown(self, namelist, drill=True, max=None):
if not namelist:
return []
result = []
for symbol in self.symbols:
found = symbol._lookup_drilldown(namelist, drill=drill, max=max)
result.extend(found)
if max and len(result) >= max:
break
return result
def resolve(self, qualname):
return self.lookup(qualname)
def filter(self, condition):
return functools.reduce(
operator.iadd,
(symbol.filter(condition) for symbol in self.symbols),
[],
)
def filter_first(self, condition):
for symbol in self.symbols:
result = symbol.filter_first(condition)
if result:
return result
return None
def all(self):
return self.filter(lambda: True)
def asjson(self):
return asjson(self)
def __repr__(self):
return f'{type(self).__name__}[]'
def __json__(self, seen=None):
return {
name: asjson(symbols, seen=seen)
for name, symbols in self.entries.items()
}
class SymbolTable(Namespace):
def add_reference(self, qualname, from_node):
symbol = self.lookup(qualname)
symbol.add_reference(qualname, from_node)
class Symbol(Namespace):
def __init__(self, name, node, ignorecase=False, duplicates=False):
super().__init__(ignorecase=ignorecase, duplicates=duplicates)
if not isinstance(name, str):
raise TypeError(f'"{name}" is not a valid symbol name')
self.name = name
self._node = node
self._parent = None
self._references = []
@property
def node(self):
return self._node
@property
def line(self):
return self.node.line
@property
def endline(self):
return self.node.endline
@property
def parent(self):
if self._parent is not None:
return self._parent()
return None
@property
def references(self):
return self._references
def insert(self, symbol):
super().insert(symbol)
symbol._parent = weakref.ref(self)
def qualpath(self):
if self.parent:
return [*self.parent.qualpath(), self.name]
return [self.name]
def qualname(self, sep=DEFAULT_SEPARATOR):
return sep.join(self.qualpath())
def _lookup_drilldown(self, namelist, drill=True, max=None):
if self.ignorecase and [self.name.upper()] == namelist:
return [self]
elif not self.ignorecase and [self.name] == namelist:
return [self]
elif self.name == namelist[0]:
return super()._lookup_drilldown(
namelist[1:], drill=drill, max=max,
)
elif drill:
return super()._lookup_drilldown(namelist, drill=drill, max=max)
else:
return []
def resolve(self, qualname):
return (
self.lookup(qualname) or
(self.parent and self.parent.resolve(qualname))
)
def filter(self, condition):
this_case = [self] if condition(self) else []
return this_case + super().filter(condition)
def filter_first(self, condition):
if condition(self):
return self
return super().filter(condition)
def add_reference(self, qualname, node):
# reference = SymbolReference(self, qualname, node)
reference = node
if reference not in self.references:
self._references.append(reference)
def line_index(self, include_entries=False, include_references=False):
result = set(self.node.line_index())
if include_references:
result.update(self.reference_line_index())
if include_entries:
for s in self.symbols:
index = s.line_index(
include_entries=include_entries,
include_references=include_references,
)
result.update(index)
assert isinstance(result, set)
assert all(isinstance(i, LineIndexInfo) for i in result)
return sorted(result)
def reference_line_index(self):
result = set()
for r in self.references:
result.update(r.line_index())
assert isinstance(result, set)
assert all(isinstance(i, LineIndexInfo) for i in result)
return result
def __repr__(self):
return f'{self.name}[]'
def __json__(self, seen=None):
return dict(
('node', type(self.node).__name__),
('entries', super().__json__(seen=seen)),
('references', asjson(self._references, seen=seen)),
)
def __getstate__(self):
state = self.__dict__.copy()
state.update(_parent=self.parent)
return state
def __setstate__(self, state):
self.__dict__.update(state)
if self._parent is not None:
self._parent = weakref.ref(self._parent)
class BasedSymbol(Namespace):
def __init__(self, name, node, duplicates=False):
super().__init__(duplicates=duplicates)
self._bases = []
@property
def bases(self):
return self._bases
def add_base(self, base):
assert isinstance(base, Symbol)
self._bases.append(base)
def _lookup_drilldown(self, namelist, drill=True, max=max):
result = super()._lookup_drilldown(namelist, drill=drill, max=max)
if result:
return result
for base in self.bases:
result = base._lookup_drilldown(namelist, drill=drill, max=max)
if result:
return result
return result
def __json__(self, seen=None):
result = super().__json__(seen=seen)
result['bases'] = asjson([b.qualname() for b in self.bases], seen=seen)
return result
@@ -0,0 +1,31 @@
from collections.abc import Mapping
from typing import Any
__REGISTRY: Mapping[str, Any] = vars()
class _Synthetic:
def __reduce__(self):
return (
synthesize(type(self).__name__, type(self).__bases__),
(),
vars(self),
)
def synthesize(name, bases):
typename = f'{__name__}.{name}'
if not isinstance(bases, tuple):
bases = (bases,)
if _Synthetic not in bases:
bases += (_Synthetic,)
constructor = __REGISTRY.get(typename)
if not constructor:
constructor = type(name, bases, {})
typename = constructor.__name__
__REGISTRY[typename] = constructor
return constructor
@@ -0,0 +1,69 @@
from .exceptions import ParseError
from .util._common import _prints
class Tokenizer:
def __init__(self, *args, **kwargs):
super().__init__()
def error(self, *args, **kwargs):
raise ParseError(_prints(*args, **kwargs))
@property
def filename(self):
raise NotImplementedError
@property
def ignorecase(self):
raise NotImplementedError
@property
def pos(self):
raise NotImplementedError
def goto(self, pos):
raise NotImplementedError
def atend(self):
raise NotImplementedError
def ateol(self):
raise NotImplementedError
@property
def current(self):
raise NotImplementedError
@property
def token(self):
return self.current
def next(self):
raise NotImplementedError
def next_token(self):
raise NotImplementedError
def match(self, token):
raise NotImplementedError
def matchre(self, pattern):
raise NotImplementedError
def posline(self, pos):
raise NotImplementedError
def line_info(self, pos=None):
raise NotImplementedError
def get_lines(self, start=None, end=None):
raise NotImplementedError
def lookahead(self):
raise NotImplementedError
def lookahead_pos(self):
if self.atend():
return ''
info = self.line_info()
return '~%d:%d' % (info.line + 1, info.col + 1)
@@ -0,0 +1,356 @@
"""
Parse and translate an EBNF grammar into a Python parser for
the described language.
"""
# ruff: noqa: PLR0912
from __future__ import annotations
import argparse
import codecs
import importlib
import sys
from pathlib import Path
from ._version import __version__
from .exceptions import ParseException
from .infos import ParserConfig
from .ngcodegen import codegen as ngpythoncg
from .ngcodegen import objectmodel as ngobjectmodel
from .parser import GrammarGenerator
from .semantics import ModelBuilderSemantics
from .util import eval_escapes
DESCRIPTION = (
'TatSu takes a grammar'
' in a variation of EBNF as input, and outputs a memoizing'
' PEG/Packrat parser in Python.'
)
def parse_args():
argparser = argparse.ArgumentParser(
prog='tatsu', description=DESCRIPTION, add_help=False,
)
main_mode = argparser.add_mutually_exclusive_group()
main_mode.add_argument(
'--generate-parser',
help='generate parser code from the grammar (default)',
action='store_true',
)
main_mode.add_argument(
'--draw',
'-d',
help='generate a diagram of the grammar (requires --outfile)',
action='store_true',
)
main_mode.add_argument(
'--object-model',
'-g',
help='generate object model from the class names given as rule arguments',
action='store_true',
)
main_mode.add_argument(
'--pretty',
'-p',
help='generate a prettified version of the input grammar',
action='store_true',
)
main_mode.add_argument(
'--pretty-lean',
help='like --pretty, but without name: or ::Parameter annotations',
action='store_true',
)
ebnf_opts = argparser.add_argument_group('parse-time options')
argparser.add_argument(
'filename',
metavar='GRAMMAR',
help='the filename of the TatSu grammar to parse',
)
ebnf_opts.add_argument(
'--color',
'-c',
help='use color in traces (requires the colorama library)',
action='store_true',
)
ebnf_opts.add_argument(
'--trace',
'-t',
help='produce verbose parsing output',
action='store_true',
)
generation_opts = argparser.add_argument_group('generation options')
generation_opts.add_argument(
'--no-left-recursion',
'-l',
help='turns left-recursion support off',
dest='left_recursion',
action='store_false',
default=True,
)
generation_opts.add_argument(
'--name',
'-m',
metavar='NAME',
help='Name for the grammar (defaults to GRAMMAR base name)',
)
generation_opts.add_argument(
'--no-nameguard',
'-n',
help='allow tokens that are prefixes of others',
dest='nameguard',
action='store_false',
default=None, # None allows grammar specified
)
generation_opts.add_argument(
'--outfile',
'--output',
'-o',
metavar='FILE',
help='output file (default is stdout)',
)
generation_opts.add_argument(
'--object-model-outfile',
'-G',
metavar='FILE',
help='generate object model and save to FILE',
)
generation_opts.add_argument(
'--whitespace',
'-w',
metavar='CHARACTERS',
help='characters to skip during parsing (use "" to disable)',
)
def import_class(path):
try:
spath = path.rsplit('.', 1)
module = importlib.import_module(spath[0])
return getattr(module, spath[1])
except Exception as e:
raise argparse.ArgumentTypeError(
f"Couldn't find class {path}",
) from e
generation_opts.add_argument(
'--base-type',
metavar='CLASSPATH',
help='class to use as base type for the object model, for example "mymodule.MyNode"',
type=import_class,
)
std_args = argparser.add_argument_group('common options')
std_args.add_argument(
'--help', '-h', help='show this help message and exit', action='help',
)
std_args.add_argument(
'--version',
'-V',
help='provide version information and exit',
action='version',
version=__version__,
)
args = argparser.parse_args()
if args.draw and not args.outfile:
argparser.error('--draw requires --outfile')
return args
__compiled_grammar_cache = {} # type: ignore[var-annotated]
def compile(
grammar,
name=None,
semantics=None,
asmodel=False,
config: ParserConfig | None = None,
**settings,
):
cache = __compiled_grammar_cache
key = (name, grammar, id(semantics))
if key in cache:
model = cache[key]
else:
gen = GrammarGenerator(name, config=config, **settings)
model = cache[key] = gen.parse(grammar, config=config, **settings)
if semantics is not None:
model.semantics = semantics
elif asmodel:
model.semantics = ModelBuilderSemantics()
return model
def parse(
grammar,
input,
start=None,
name=None,
semantics=None,
asmodel=False,
config: ParserConfig | None = None,
**settings,
):
model = compile(
grammar,
name=name,
semantics=semantics,
asmodel=asmodel,
config=config,
**settings,
)
return model.parse(
input, start=start, semantics=semantics, config=config, **settings,
)
def to_python_sourcecode(
grammar,
name=None,
filename=None,
config: ParserConfig | None = None,
**settings,
):
model = compile(
grammar, name=name, filename=filename, config=config, **settings,
)
return ngpythoncg(model)
def to_python_model(
grammar,
name=None,
filename=None,
base_type=None,
config: ParserConfig | None = None,
**settings,
):
model = compile(
grammar, name=name, filename=filename, config=config, **settings,
)
return ngobjectmodel.modelgen(model, base_type=base_type)
# for backwards compatibility. Use `compile()` instead
def genmodel(
name=None,
grammar=None,
semantics=None,
config: ParserConfig | None = None,
**settings,
):
if grammar is None:
raise ParseException('grammar is None')
return compile(
grammar, name=name, semantics=semantics, config=config, **settings,
)
def gencode(
name=None,
grammar=None,
trace=False,
filename=None,
codegen=ngpythoncg,
config: ParserConfig | None = None,
**settings,
):
model = compile(
grammar,
name=name,
filename=filename,
trace=trace,
config=config,
**settings,
)
return codegen(model)
def prepare_for_output(filename):
if filename:
filename = Path(filename)
if filename.is_file():
filename.unlink()
dirname = filename.parent
if dirname.exists():
dirname.mkdir(parents=True, exist_ok=True)
def save(filename, content):
with codecs.open(filename, 'w', encoding='utf-8') as f:
f.write(content)
def main():
args = parse_args()
if args.whitespace:
args.whitespace = eval_escapes(args.whitespace)
outfile = args.outfile
prepare_for_output(outfile)
prepare_for_output(args.object_model_outfile)
grammar = codecs.open(args.filename, encoding='utf-8').read()
try:
model = compile(
grammar,
args.name,
trace=args.trace,
filename=args.filename,
colorize=args.color,
)
model.whitespace = args.whitespace
model.nameguard = args.nameguard
model.left_recursion = args.left_recursion
if args.draw:
from tatsu import diagrams
diagrams.draw(outfile, model)
else:
if args.pretty:
result = model.pretty()
elif args.pretty_lean:
result = model.pretty_lean()
elif args.object_model:
result = ngobjectmodel.modelgen(model, base_type=args.base_type)
else:
result = ngpythoncg(model)
if outfile:
save(outfile, result)
else:
print(result)
# if requested, always save it
if args.object_model_outfile:
save(
args.object_model_outfile,
ngobjectmodel.modelgen(model, base_type=args.base_type),
)
print('-' * 72, file=sys.stderr)
print(
f'{len(grammar.split()):12,d} lines in grammar', file=sys.stderr,
)
print(f'{len(model.rules):12,d} rules in grammar', file=sys.stderr)
print(f'{model.nodecount():12,d} nodes in AST', file=sys.stderr)
except ParseException as e:
print(e, file=sys.stderr)
sys.exit(1)
if __name__ == '__main__':
main()
@@ -0,0 +1 @@
from ._common import * # noqa: F403
@@ -0,0 +1,540 @@
from __future__ import annotations
import codecs
import datetime
import enum
import functools
import json
import keyword
import logging
import operator
import os
import os.path
import re
import sys
import warnings
import weakref
from collections.abc import Iterable, Mapping, MutableSequence
from io import StringIO
from itertools import zip_longest
from pathlib import Path
logger = logging.getLogger('tatsu')
logger.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
formatter = logging.Formatter('%(message)s')
ch.setFormatter(formatter)
logger.addHandler(ch)
ESCAPE_SEQUENCE_RE = re.compile(
r"""
( \\U........ # 8-digit Unicode escapes
| \\u.... # 4-digit Unicode escapes
| \\x.. # 2-digit Unicode escapes
| \\[0-7]{1,3} # Octal character escapes
| \\N\{[^}]+\} # Unicode characters by name
| \\[\\'"abfnrtv] # Single-character escapes
)""",
re.UNICODE | re.VERBOSE,
)
class AsJSONMixin:
def __json__(self, seen=None):
return {
'__class__': type(self).__name__,
**asjson(self._pubdict(), seen=seen),
}
def _pubdict(self):
return {
name: value
for name, value in vars(self).items()
if not name.startswith('_')
}
def program_name():
import __main__ as main
if package := main.__package__:
return package
return Path(main.__file__).name
def is_posix():
return os.name == 'posix'
def _prints(*args, **kwargs):
with StringIO() as f:
kwargs['file'] = f
kwargs['end'] = ''
print(*args, **kwargs)
return f.getvalue()
def info(*args, **kwargs):
logger.info(_prints(*args, **kwargs))
def debug(*args, **kwargs):
logger.debug(_prints(*args, **kwargs))
def warning(*args, **kwargs):
logger.warning(_prints(*args, **kwargs))
def error(*args, **kwargs):
logger.error(_prints(*args, **kwargs))
def identity(*args):
if len(args) == 1:
return args[0]
return args
def is_list(o):
return type(o) is list
def to_list(o):
if o is None:
return []
elif isinstance(o, list):
return o
else:
return [o]
def is_namedtuple(obj) -> bool:
return (
len(type(obj).__bases__) == 1
and isinstance(obj, tuple)
and hasattr(obj, '_asdict')
and hasattr(obj, '_fields')
and all(isinstance(f, str) for f in getattr(obj, '_fields', ()))
)
def simplify_list(x):
if isinstance(x, list) and len(x) == 1:
return simplify_list(x[0])
return x
def extend_list(x, n, default=None):
def _null():
pass
default = default or _null
missing = max(0, 1 + n - len(x))
x.extend(default() for _ in range(missing))
def contains_sublist(lst, sublst):
n = len(sublst)
return any(sublst == lst[i: i + n] for i in range(1 + len(lst) - n))
def join_lists(lists):
return functools.reduce(operator.iadd, lists, [])
def flatten(o):
if not isinstance(o, MutableSequence):
yield o
return
for item in o:
yield from flatten(item)
def compress_seq(seq):
seen = set()
result = []
for x in seq:
if x not in seen:
result.append(x)
seen.add(x)
return result
def eval_escapes(s):
"""
Given a string, evaluate escape sequences starting with backslashes as
they would be evaluated in Python source code. For a list of these
sequences, see: https://docs.python.org/3/reference/lexical_analysis.html
This is not the same as decoding the whole string with the 'unicode-escape'
codec, because that provides no way to handle non-ASCII characters that are
literally present in the string.
"""
# by Rob Speer
def decode_match(match):
return codecs.decode(match.group(0), 'unicode-escape')
return ESCAPE_SEQUENCE_RE.sub(decode_match, s)
def isiter(value):
return isinstance(value, Iterable) and not isinstance(
value, str | bytes | bytearray,
)
def trim(text, tabwidth=4):
"""
Trim text of common, leading whitespace.
Based on the trim algorithm of PEP 257:
http://www.python.org/dev/peps/pep-0257/
"""
if not text:
return ''
lines = text.expandtabs(tabwidth).splitlines()
maxindent = len(text)
indent = maxindent
for line in lines[1:]:
stripped = line.lstrip()
if stripped:
indent = min(indent, len(line) - len(stripped))
trimmed = [lines[0].strip()] + [
line[indent:].rstrip() for line in lines[1:]
]
i = 0
while i < len(trimmed) and not trimmed[i]:
i += 1
return '\n'.join(trimmed[i:])
def indent(text, indent=1, multiplier=4):
"""Indent the given block of text by indent*4 spaces"""
if text is None:
return ''
text = str(text)
if indent >= 0:
sindent = ' ' * multiplier * indent
text = '\n'.join((sindent + t).rstrip() for t in text.splitlines())
return text
def format_if(fmt, values):
return fmt % values if values else ''
def notnone(value, default=None):
return value if value is not None else default
def timestamp():
return '.'.join(
'%2.2d' % t for t in datetime.datetime.utcnow().utctimetuple()[:-2]
)
def asjson(obj, seen=None): # noqa: PLR0911, PLR0912
if obj is None or isinstance(obj, int | float | str | bool):
return obj
if seen is None:
seen = set()
elif id(obj) in seen:
return f'{type(obj).__name__}@{id(obj)}'
if isinstance(obj, Mapping | AsJSONMixin) or isiter(obj):
seen.add(id(obj))
try:
if isinstance(obj, weakref.ReferenceType | weakref.ProxyType):
return f'{obj.__class__.__name__}@0x{hex(id(obj)).upper()[2:]}'
elif hasattr(obj, '__json__'):
return obj.__json__(seen=seen)
elif is_namedtuple(obj):
return asjson(obj._asdict(), seen=seen)
elif isinstance(obj, Mapping):
result = {}
for k, v in obj.items():
try:
result[k] = asjson(v, seen=seen)
except TypeError:
debug('Unhashable key?', type(k), str(k))
raise
return result
elif isiter(obj):
return [asjson(e, seen=seen) for e in obj]
elif isinstance(obj, enum.Enum):
return asjson(obj.value)
else:
return repr(obj)
finally:
# NOTE: id()s may be reused
# https://docs.python.org/3/library/functions.html#id
seen -= {id(obj)}
def minjson(obj, typesfiltered=(str, list, dict)):
if isinstance(obj, Mapping):
return {
name: minjson(value)
for name, value in obj.items()
if (
not name.startswith('_')
and value is not None
and (value or not isinstance(value, typesfiltered))
)
}
elif isiter(obj):
return [minjson(e) for e in obj]
else:
return obj
def plainjson(obj):
if isinstance(obj, Mapping):
return {
name: plainjson(value)
for name, value in obj.items()
if name not in {'__class__', 'parseinfo'}
}
elif isinstance(obj, weakref.ReferenceType | weakref.ProxyType):
return '@ref'
elif isinstance(obj, str) and obj.startswith('@'):
return '@ref'
elif isiter(obj):
return [plainjson(e) for e in obj]
else:
return obj
class FallbackJSONEncoder(json.JSONEncoder):
"""A JSON Encoder that falls back to repr() for non-JSON-aware objects."""
def default(self, o):
return repr(o)
def asjsons(obj):
return json.dumps(asjson(obj), indent=2, cls=FallbackJSONEncoder)
def prune_dict(d, predicate):
"""Remove all items x where predicate(x, d[x])"""
keys = [k for k, v in d.items() if predicate(k, v)]
for k in keys:
del d[k]
def safe_name(name):
if keyword.iskeyword(name):
return name + '_'
return name
def chunks(iterable, size, fillvalue=None):
return zip_longest(*[iter(iterable)] * size, fillvalue=fillvalue)
def generic_main(custom_main, parser_class, name='Unknown'):
import argparse
class ListRules(argparse.Action):
def __call__(self, parser, namespace, values, option_string=None):
print('Rules:')
for r in parser_class.rule_list():
print(r)
print()
sys.exit(0)
argp = argparse.ArgumentParser(description=f'Simple parser for {name}.')
addarg = argp.add_argument
addarg(
'-c',
'--color',
help='use color in traces (requires the colorama library)',
action='store_true',
)
addarg(
'-l',
'--list',
action=ListRules,
nargs=0,
help='list all rules and exit',
)
addarg(
'-n',
'--no-nameguard',
action='store_true',
dest='no_nameguard',
help="disable the 'nameguard' feature",
)
addarg(
'-t', '--trace', action='store_true', help='output trace information',
)
addarg(
'-w',
'--whitespace',
type=str,
default=None,
help='whitespace specification',
)
addarg(
'file',
metavar='FILE',
help="the input file to parse or '-' for standard input",
nargs='?',
default='-',
)
addarg(
'startrule',
metavar='STARTRULE',
nargs='?',
help='the start rule for parsing',
default=None,
)
args = argp.parse_args()
try:
return custom_main(
args.file,
start=args.startrule,
trace=args.trace,
whitespace=args.whitespace,
nameguard=not args.no_nameguard,
colorize=args.color,
)
except KeyboardInterrupt:
pass
# decorator
def deprecated(fun):
@functools.wraps(fun)
def wrapper(*args, **kwargs):
warnings.filterwarnings('default', category=DeprecationWarning)
msg = 'Call to deprecated function {}.'
warnings.warn(
msg.format(fun.__name__), category=DeprecationWarning, stacklevel=2,
)
return fun(*args, **kwargs)
return wrapper
def left_assoc(elements):
if not elements:
return ()
it = iter(elements)
expre = next(it)
for e in it:
op = e
expre = (op, expre, next(it))
return expre
def right_assoc(elements):
if not elements:
return ()
def assoc(it):
left = next(it)
try:
op = next(it)
except StopIteration:
return left
else:
return (op, left, assoc(it))
return assoc(iter(elements))
try:
import psutil
except ImportError:
def memory_use():
return 0
else:
def memory_use():
process = psutil.Process(os.getpid())
return process.memory_info().rss
def try_read(filename):
if isinstance(filename, Path):
filename = str(filename)
for e in ['utf-16', 'utf-8', 'latin-1', 'cp1252', 'ascii']:
try:
with Path(filename).open(encoding=e) as f:
return f.read()
except UnicodeError:
pass
raise UnicodeDecodeError(f"cannot find the encoding for '{filename}'")
def filelist_from_patterns(patterns, ignore=None, base='.', sizesort=False):
base = Path(base or '.').expanduser()
filenames = set()
for pattern in patterns or []:
path = base / pattern
if path.is_file():
filenames.add(path)
continue
if path.is_dir():
path += '/*'
parts = path.parts[1:] if path.is_absolute() else path.parts
joined_pattern = str(Path().joinpath(*parts))
filenames.update(
p for p in Path(path.root).glob(joined_pattern) if not p.is_dir()
)
filenames = list(filenames)
def excluded(path):
if any(path.match(ex) for ex in ignore):
return True
return any(
any(Path(part).match(ex) for ex in ignore) for part in path.parts
)
if ignore:
filenames = [path for path in filenames if not excluded(path)]
if sizesort:
filenames.sort(key=lambda f: f.stat().st_size)
return filenames
def short_relative_path(path, base='.'):
path = Path(path)
base = Path(base)
common = Path(os.path.commonpath([base.resolve(), path.resolve()]))
if common == path.root:
return path
elif common == Path.home():
up = Path('~')
elif common == base:
up = Path()
else:
n = len(base.parts) - len(common.parts)
up = Path('../' * n)
rel = str(up / path.resolve().relative_to(common))
if len(rel) < len(str(path)):
return rel
else:
return str(path)
@@ -0,0 +1,117 @@
from __future__ import annotations
import re
from collections.abc import Iterable
from functools import cache
from typing import TypeVar
_T = TypeVar('_T')
_undefined = object() # unique object for when None is not a good default
def first(iterable, default=_undefined):
"""Return the first item of *iterable*, or *default* if *iterable* is
empty.
>>> first([0, 1, 2, 3])
0
>>> first([], 'some default')
'some default'
If *default* is not provided and there are no items in the iterable,
raise ``ValueError``.
:func:`first` is useful when you have a generator of expensive-to-retrieve
values and want any arbitrary one. It is marginally shorter than
``next(iter(iterable), default)``.
"""
# NOTE: https://more-itertools.readthedocs.io/en/stable/_modules/more_itertools/more.html#first
try:
return next(iter(iterable))
except StopIteration as e:
# I'm on the edge about raising ValueError instead of StopIteration. At
# the moment, ValueError wins, because the caller could conceivably
# want to do something different with flow control when I raise the
# exception, and it's weird to explicitly catch StopIteration.
if default is _undefined:
raise ValueError(
'first() was called on an empty iterable, and no '
'default value was provided.',
) from e
return default
def match_to_find(m: re.Match):
if m is None:
return None
g = m.groups(default=m.string[0:0])
if len(g) == 1:
return g[0]
else:
return g or m.group()
def findalliter(pattern, string, pos=None, endpos=None, flags=0):
"""
like finditer(), but with return values like findall()
implementation taken from cpython/Modules/_sre.c/findall()
"""
r = (
pattern
if isinstance(pattern, re.Pattern)
else re.compile(pattern, flags=flags)
)
if endpos is not None:
iterator = r.finditer(string, pos=pos, endpos=endpos)
elif pos is not None:
iterator = r.finditer(string, pos=pos)
else:
iterator = r.finditer(string)
for m in iterator:
yield match_to_find(m)
def findfirst(pattern, string, pos=None, endpos=None, flags=0, default=_undefined):
"""
Avoids using the inefficient findall(...)[0], or first(findall(...))
"""
return first(
findalliter(pattern, string, pos=pos, endpos=endpos, flags=flags),
default=default,
)
def topsort(nodes: Iterable[_T], order: Iterable[tuple[_T, _T]]) -> list[_T]:
# https://en.wikipedia.org/wiki/Topological_sorting
order = set(order)
result: list[_T] = [] # Empty list that will contain the sorted elements
def with_incoming() -> set[_T]:
return {m for (_, m) in order}
# Set of all nodes with no incoming edges
pending = list(set(nodes) - with_incoming())
while pending:
n = pending.pop()
result.append(n)
# nodes m with an edge from n to m
outgoing = {m for (x, m) in order if x == n}
order -= {(n, m) for m in outgoing}
pending.extend(outgoing - with_incoming())
if order:
raise ValueError('There are cycles in the topological order')
return list(reversed(result)) # a topologically sorted list
@cache
def cached_re_compile(regex: re.Pattern | str | bytes) -> re.Pattern | None:
if isinstance(regex, re.Pattern):
return regex
return re.compile(regex) if isinstance(regex, (str | bytes)) else None
@@ -0,0 +1,284 @@
import sys
import time
from collections import namedtuple
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed
from functools import partial
from multiprocessing import Pool, cpu_count
from pathlib import Path
import rich
from rich.progress import (
BarColumn,
Progress,
TaskProgressColumn,
TextColumn,
TimeElapsedColumn,
TimeRemainingColumn,
)
from tatsu.util.unicode_characters import (
U_CHECK_MARK,
U_CROSSED_SWORDS,
)
from ..util import identity, memory_use, program_name, try_read
__all__ = ['parallel_proc', 'processing_loop']
ERROR_LOG_FILENAME = 'ERROR.log'
EOLCH = '\r' if sys.stderr.isatty() else '\n'
sys.setrecursionlimit(2**16)
__Task = namedtuple('__Task', 'payload args kwargs')
class ParprocResult:
def __init__(self, payload):
self.payload = payload
self.outcome = None
self.exception = None
self.linecount = 0
self.time = 0
self.memory = 0
@property
def success(self):
return self.exception is None
def __str__(self):
return str(self.__dict__)
def process_payload(process, task, pickable=identity, reraise=False):
start_time = time.process_time()
result = ParprocResult(task.payload)
try:
outcome = process(task.payload, *task.args, **task.kwargs)
result.memory = memory_use()
if hasattr(outcome, 'linecount'):
result.linecount = outcome.linecount
else:
result.linecount = len(try_read(task.payload).splitlines())
result.outcome = pickable(outcome)
except KeyboardInterrupt:
return None
except Exception as e:
result.exception = e
finally:
result.time = time.process_time() - start_time
return result
def _executor_pmap(executor, process, tasks):
nworkers = max(1, cpu_count())
n = nworkers * 8
chunks = [tasks[i:i + n] for i in range(0, len(tasks), n)]
for chunk in chunks:
with executor(max_workers=nworkers) as ex:
futures = [ex.submit(process, task) for task in chunk]
for future in as_completed(futures):
yield future.result()
def _thread_pmap(process, tasks):
yield from _executor_pmap(ThreadPoolExecutor, process, tasks)
def _process_pmap(process, tasks):
yield from _executor_pmap(ProcessPoolExecutor, process, tasks)
def _imap_pmap(process, tasks):
nworkers = max(1, cpu_count())
n = nworkers * 4
chunks = [tasks[i:i + n] for i in range(0, len(tasks), n)]
count = sum(len(c) for c in chunks)
if len(tasks) != count:
raise RuntimeError('number of chunked tasks different %d != %d' % (len(tasks), count))
for chunk in chunks:
with Pool(processes=nworkers) as pool:
yield from pool.imap_unordered(process, chunk)
_pmap = _imap_pmap
def parallel_proc(payloads, process, *args, **kwargs):
pickable = kwargs.pop('pickable', identity)
parallel = kwargs.pop('parallel', True)
reraise = kwargs.pop('reraise', False)
process = partial(process_payload, process, pickable=pickable, reraise=reraise)
tasks = [__Task(payload, args, kwargs) for payload in payloads]
try:
if len(tasks) == 1:
yield process(tasks[0])
else:
pmap = _pmap if parallel else map
yield from pmap(process, tasks)
except KeyboardInterrupt:
return
def _build_progressbar(total):
progress = Progress(
TextColumn(f"[progress.description]{program_name()}"),
BarColumn(),
# *Progress.get_default_columns(),
TaskProgressColumn(),
TimeElapsedColumn(),
TimeRemainingColumn(),
TextColumn("[progress.description]{task.description}"),
refresh_per_second=1,
speed_estimate_period=30.0,
)
task = progress.add_task('', total=total)
return progress, task
def processing_loop(filenames, process, *args, reraise=False, **kwargs): # pylint: disable=too-many-locals
try:
total = len(filenames)
total_time = 0
run_time = 0
start_time = time.time()
results = parallel_proc(filenames, process, *args, **kwargs)
results = results or []
count = 0
success_count = 0
success_linecount = 0
progress, progress_task = _build_progressbar(total)
if total == 1:
log = sys.stderr
else:
log = Path(ERROR_LOG_FILENAME).open('w')
with progress:
for result in results:
if result is None:
continue
count += 1
total_time = time.time() - start_time
filename = Path(result.payload).name
if result.exception:
icon = f'[red]{U_CROSSED_SWORDS}'
else:
icon = f'[green]{U_CHECK_MARK}'
progress.update(
progress_task,
advance=1,
description=f'{icon} {filename}',
)
if result.exception:
print(file=log)
try:
print('ERROR:', result.payload, file=log)
print(result.exception, file=log)
except Exception:
# in case of errors while serializing the exception
print('EXCEPTION', type(result.exception).__name__, file=log)
if reraise:
raise result.exception
elif result.outcome is not None:
success_count += 1
success_linecount += result.linecount
run_time += result.time
yield result
log.flush()
progress.update(progress_task, advance=0, description='')
progress.stop()
file_process_summary(
filenames, total_time, run_time, success_count, success_linecount, log,
)
except KeyboardInterrupt:
return
def file_process_progress(latest_result, count, total, total_time):
filename = latest_result.payload
percent = count / total
mb_memory = (latest_result.memory + memory_use()) // (1024 * 1024)
eta = (total - count) * 0.8 * total_time / (0.2 * count)
bar = '[%-24s]' % ('#' * round(24 * percent))
print(
'%3d/%-3d' % (count, total),
bar,
'%3d%%' % (100 * percent),
# format_hours(total_time),
f'{format_hours(eta)}ETA',
format_minutes(latest_result),
'%3dMiB' % mb_memory if mb_memory else '',
(Path(filename).name + ' ' * 80)[:40],
file=sys.stderr,
end=EOLCH)
def format_minutes(result):
return f'{result.time / 60:3.0f}:{result.time % 60:04.1f}'
def format_hours(time):
return f'{time // 3600:2.0f}:{(time // 60) % 60:02.0f}:{time % 60:02.0f}'
def file_process_summary(
filenames, total_time, run_time, success_count, success_linecount, log,
):
filecount = 0
linecount = 0
for fname in filenames:
filecount += 1
nlines = len(try_read(fname).splitlines())
linecount += nlines
failures = filecount - success_count
summary_text = '''\
-----------------------------------------------------------------------
{:12,d} files input
{:12,d} source lines input
{:12,d} total lines processed
{:12,d} successes
{:12,d} failures
{:12.1f}% success rate
{:>13s} time
{:>13s} run time
'''
summary_text = '\n'.join(s.strip() for s in summary_text.splitlines())
summary = summary_text.format(
filecount,
linecount,
success_linecount,
success_count,
failures,
100 * success_count / filecount if filecount != 0 else 0,
format_hours(total_time),
format_hours(run_time),
)
print(EOLCH + 80 * ' ', file=log)
print(summary, file=log)
print(EOLCH + 80 * ' ', file=sys.stderr)
if failures:
rich.print(f'[red bold]FAILURES logged to [green]{log.name}!')
print(file=sys.stderr)
if log != sys.stderr:
print(summary, file=sys.stderr)
if failures:
sys.exit(1)
@@ -0,0 +1,85 @@
import argparse
from . import filelist_from_patterns
from .parproc import processing_loop
def parallel_test_run(parse, options):
def pysearch(pattern):
if pattern.endswith('.py'):
return pattern
else:
if not pattern.endswith('/'):
pattern += '/'
if '**' not in pattern:
pattern += '**/'
return pattern + '*.py'
try:
patterns = [pysearch(p) for p in options.patterns]
filenames = filelist_from_patterns(
patterns, sizesort=options.sort, ignore=options.ignore,
)
kwargs = vars(options)
kwargs.pop('patterns', None)
kwargs.pop('sort', None)
kwargs.pop('ignore', None)
parallel = not kwargs.pop('serial', False)
return processing_loop(parse, filenames, parallel=parallel, **kwargs)
except KeyboardInterrupt:
if options.verbose:
raise
def parse_args():
argparser = argparse.ArgumentParser(
prog=__package__,
# description=DESCRIPTION,
add_help=False,
)
argparser.add_argument(
'patterns', nargs='+', metavar='PATTERNS', help='filename patterns',
)
argparser.add_argument(
'--help', '-h', help='show this help message and exit', action='help',
)
argparser.add_argument(
'--ignore',
'-i',
metavar='PATTERN',
help='ignore these patterns',
action='append',
)
argparser.add_argument(
'--sort', '-s', help='sort files by size', action='store_true',
)
argparser.add_argument(
'--serial', '-S', help='do not run in parallel', action='store_true',
)
argparser.add_argument(
'--trace',
'-t',
help='produce verbose output for a parse',
action='store_true',
)
argparser.add_argument(
'--verbose',
'-v',
help='output exceptions as they happen',
action='store_true',
)
argparser.add_argument(
'--exitfirst',
'-x',
help='output exceptions as they happen',
action='store_true',
)
return argparser.parse_args()
def generic_main(parse):
return parallel_test_run(parse, parse_args())
@@ -0,0 +1,54 @@
from __future__ import annotations
from tatsu.util import is_posix
U_LARROW = '\u2190'
U_DLARROW = '\u2199'
U_LDARROW = '\u21D0'
U_UDARROW = '\u21D1'
U_RDARROW = '\u21D2'
U_DDARROW = '\u21D3'
U_L_TRIPPLE_ARROW = '\u21DA'
U_WARNING = '\u26A0'
U_NOT_EQUAL_TO = '\u2260'
U_IDENTICAL_TO = '\u2261'
U_NOT_IDENTICAL_TO = '\u2262'
U_CHECK_MARK = '\u2713'
U_POWER_SYMBOL = '\u23FB'
U_POWER_ON_SYMBOL = '\u23FC'
U_POWER_OFF_SYMBOL = '\u23FD'
U_GREEK_SMALL_LETTER_EPSILON = '\u03B5'
U_REGISTERED_SIGN = '\u00AE'
U_RIENNMAN = '\u211D'
U_ANTICLOCKWISE_OPEN_CIRCLE_ARROW = '\u21BA'
U_ANTICLOCKWISE_GAPPED_CIRCLE_ARROW = '\u27F2'
U_PUNCTUATION_SPACE = '\u2008'
U_FOUR_PER_EM_SPACE = '\u2005'
U_MEDIUM_MATHEMATICAL_SPACE = '\u205F'
U_ZERO_WIDTH_NO_BREAK_SPACE = '\uFEFF'
U_BLACK_SCISSORS = '\u2702'
U_CROSSED_SWORDS = '\u2694'
if not is_posix():
C_DERIVE = '<'
C_ENTRY = '<'
C_SUCCESS = '>'
C_FAILURE = '!'
C_RECURSION = 'r '
C_CUT = '~'
else:
C_DERIVE = U_DLARROW
C_ENTRY = C_DERIVE
C_SUCCESS = U_IDENTICAL_TO
C_FAILURE = U_NOT_IDENTICAL_TO
C_RECURSION = U_ANTICLOCKWISE_GAPPED_CIRCLE_ARROW + U_FOUR_PER_EM_SPACE
C_CUT = U_CROSSED_SWORDS
@@ -0,0 +1,174 @@
from __future__ import annotations
import re
from collections.abc import Mapping
from contextlib import contextmanager
from typing import Any
from .objectmodel import Node
from .util import is_list
class NodeWalkerMeta(type):
def __new__(mcs, name, bases, dct):
class_ = super().__new__(mcs, name, bases, dct)
class_._walker_cache = {}
return class_
class NodeWalker(metaclass=NodeWalkerMeta):
def __init__(self):
super().__init__()
# copy the class attribute to avoid linter warnings
self._walker_cache = type(
self,
)._walker_cache # pylint: disable=no-member
def walk(self, node: Node | list[Node], *args, **kwargs) -> Any:
if isinstance(node, list | tuple):
return [self.walk(n, *args, **kwargs) for n in node]
if isinstance(node, Mapping):
return {
name: self.walk(value, *args, **kwargs)
for name, value in node.items()
}
walker = self._find_walker(node)
if callable(walker):
return walker(self, node, *args, **kwargs)
else:
return node
def walk_children(self, node: Node, *args, **kwargs):
if not isinstance(node, Node):
return []
return [
self.walk(child, *args, **kwargs)
for child in node.children()
]
# note: backwards compatibility
_walk_children = walk_children
def _find_walker(self, node: Node, prefix='walk_'):
def pythonize_match(m):
return '_' + m.group().lower()
cls = self.__class__
node_cls = node.__class__
node_cls_qualname = node_cls.__qualname__
if walker := self._walker_cache.get(node_cls_qualname):
return walker
node_classes = [node.__class__]
while node_classes:
node_cls = node_classes.pop(0)
cammelcase_name = node_cls.__name__
walker = getattr(cls, prefix + cammelcase_name, None)
if callable(walker):
break
# walk__pythonic_name with double underscore after walk
pythonic_name = re.sub(
r'[A-Z]+', pythonize_match, node_cls.__name__,
)
if pythonic_name != cammelcase_name:
walker = getattr(cls, prefix + pythonic_name, None)
if callable(walker):
break
# walk_pythonic_name with single underscore after prefix
pythonic_name = pythonic_name.lstrip('_')
if pythonic_name != cammelcase_name:
walker = getattr(cls, prefix + pythonic_name, None)
if callable(walker):
break
for b in node_cls.__bases__:
if b not in node_classes:
node_classes.append(b)
else:
walker = getattr(cls, '_walk_default', None)
if walker is None:
walker = getattr(
cls, 'walk_default', None,
) # backwards compatibility
if not callable(walker):
walker = None
self._walker_cache[node_cls_qualname] = walker
return walker
class PreOrderWalker(NodeWalker):
def walk(self, node, *args, **kwargs):
result = super().walk(node, *args, **kwargs)
if result is not None:
self.walk_children(node, *args, **kwargs)
return result
class DepthFirstWalker(NodeWalker):
def walk(self, node, *args, **kwargs):
if isinstance(node, Node):
children = [self.walk(c, *args, **kwargs) for c in node.children()]
return super().walk(node, children, *args, **kwargs)
elif isinstance(node, Mapping):
return {n: self.walk(e, *args, **kwargs) for n, e in node.items()}
elif is_list(node):
return [self.walk(e, *args, **kwargs) for e in iter(node)]
else:
return super().walk(node, [], *args, **kwargs)
class ContextWalker(NodeWalker):
def __init__(self, initial_context):
super().__init__()
self._initial_context = initial_context
self._context_stack = [initial_context]
# abstract
def get_node_context(self, node, *args, **kwargs):
return node
# abstract
def enter_context(self, ctx):
pass
# abstract
def leave_context(self, ctx):
pass
def push_context(self, ctx):
self._context_stack.append(ctx)
def pop_context(self):
self._context_stack.pop()
@property
def initial_context(self):
return self._initial_context
@property
def context(self):
return self._context_stack[-1]
@contextmanager
def new_context(self, node, *args, **kwargs):
ctx = self.get_node_context(node, *args, **kwargs)
if ctx == self.context:
yield ctx
else:
self.enter_context(ctx)
try:
self.push_context(ctx)
try:
yield ctx
finally:
self.pop_context()
finally:
self.leave_context(ctx)