add proxy for the first part and try to develop the second part
This commit is contained in:
@@ -0,0 +1,21 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from ._config import __toolname__, __version__
|
||||
from .tool import ( # pylint: disable=W0622
|
||||
compile,
|
||||
main,
|
||||
parse,
|
||||
to_python_model,
|
||||
to_python_sourcecode,
|
||||
)
|
||||
|
||||
assert __version__
|
||||
assert __toolname__
|
||||
assert bool(compile)
|
||||
assert bool(parse)
|
||||
assert bool(to_python_sourcecode)
|
||||
assert bool(to_python_model)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import tatsu
|
||||
|
||||
if __name__ == '__main__':
|
||||
tatsu.main()
|
||||
@@ -0,0 +1,3 @@
|
||||
from ._version import __version__ # noqa: F401
|
||||
|
||||
__toolname__ = 'TatSu'
|
||||
@@ -0,0 +1 @@
|
||||
__version__ = '5.13.1'
|
||||
@@ -0,0 +1,134 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import operator
|
||||
from functools import reduce
|
||||
|
||||
from .util import asjson, is_list
|
||||
|
||||
|
||||
class AST(dict):
|
||||
_frozen = False
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__()
|
||||
self.update(*args, **kwargs)
|
||||
self._frozen = True
|
||||
|
||||
@property
|
||||
def frozen(self):
|
||||
return self._frozen
|
||||
|
||||
@property
|
||||
def parseinfo(self):
|
||||
try:
|
||||
return super().__getitem__('parseinfo')
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
def set_parseinfo(self, value):
|
||||
super().__setitem__('parseinfo', value)
|
||||
|
||||
def copy(self):
|
||||
return copy.copy(self)
|
||||
|
||||
def asjson(self):
|
||||
return asjson(self)
|
||||
|
||||
def _set(self, key, value, force_list=False):
|
||||
key = self._safekey(key)
|
||||
previous = self.get(key)
|
||||
|
||||
if previous is None and force_list:
|
||||
value = [value]
|
||||
elif previous is None:
|
||||
pass
|
||||
elif is_list(previous):
|
||||
value = [*previous, value]
|
||||
else:
|
||||
value = [previous, value]
|
||||
|
||||
super().__setitem__(key, value)
|
||||
|
||||
def _setlist(self, key, value):
|
||||
return self._set(key, value, force_list=True)
|
||||
|
||||
def __copy__(self):
|
||||
return AST(self)
|
||||
|
||||
def __getitem__(self, key):
|
||||
if key in self:
|
||||
return super().__getitem__(key)
|
||||
key = self._safekey(key)
|
||||
if key in self:
|
||||
return super().__getitem__(key)
|
||||
return None
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
self._set(key, value)
|
||||
|
||||
def __delitem__(self, key):
|
||||
key = self._safekey(key)
|
||||
super().__delitem__(key)
|
||||
|
||||
def __setattr__(self, name, value):
|
||||
if self._frozen and name not in vars(self):
|
||||
raise AttributeError(
|
||||
f'{type(self).__name__} attributes are fixed. '
|
||||
f' Cannot set attribute "{name}".',
|
||||
)
|
||||
super().__setattr__(name, value)
|
||||
|
||||
def __getattr__(self, name):
|
||||
key = self._safekey(name)
|
||||
if key in self:
|
||||
return self[key]
|
||||
elif name in self:
|
||||
return self[name]
|
||||
|
||||
try:
|
||||
return super().__getattribute__(name)
|
||||
except AttributeError:
|
||||
return None
|
||||
|
||||
def __hasattribute__(self, name):
|
||||
try:
|
||||
super().__getattribute__(name)
|
||||
except (TypeError, AttributeError):
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
|
||||
def __reduce__(self):
|
||||
return (AST, (list(self.items()),))
|
||||
|
||||
def _safekey(self, key):
|
||||
while self.__hasattribute__(key):
|
||||
key += '_'
|
||||
return key
|
||||
|
||||
def _define(self, keys, list_keys=None):
|
||||
for key in (self._safekey(k) for k in keys):
|
||||
if key not in self:
|
||||
super().__setitem__(key, None)
|
||||
|
||||
for key in (self._safekey(k) for k in list_keys or []):
|
||||
if key not in self:
|
||||
super().__setitem__(key, [])
|
||||
|
||||
def __json__(self, seen=None):
|
||||
return {name: asjson(value, seen=seen) for name, value in self.items()}
|
||||
|
||||
def __repr__(self):
|
||||
return repr(self.asjson())
|
||||
|
||||
def __str__(self):
|
||||
return str(self.asjson())
|
||||
|
||||
def __hash__(self):
|
||||
# NOTE: objects are actually mutable during creation
|
||||
return reduce(
|
||||
operator.xor,
|
||||
(hash((name, id(value))) for name, value in self.items()),
|
||||
0,
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,411 @@
|
||||
"""
|
||||
The Buffer class provides the functionality required by a parser-driven lexer.
|
||||
|
||||
Line analysis and caching are done so the parser can freely move with goto(p)
|
||||
to any position in the parsed text, and still recover accurate information
|
||||
about source lines and content.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from itertools import repeat, takewhile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .exceptions import ParseError
|
||||
from .infos import (
|
||||
CommentInfo,
|
||||
LineIndexInfo,
|
||||
LineInfo,
|
||||
ParserConfig,
|
||||
PosLine,
|
||||
UndefinedStr,
|
||||
)
|
||||
from .tokenizing import Tokenizer
|
||||
from .util import (
|
||||
contains_sublist,
|
||||
extend_list,
|
||||
identity,
|
||||
)
|
||||
from .util.misc import cached_re_compile, match_to_find
|
||||
|
||||
DEFAULT_WHITESPACE_RE = re.compile(r'(?m)\s+')
|
||||
|
||||
# for backwards compatibility with existing parsers
|
||||
LineIndexEntry = LineIndexInfo
|
||||
|
||||
|
||||
class Buffer(Tokenizer):
|
||||
|
||||
def __init__(
|
||||
self, text, /, config: ParserConfig | None = None, **settings: Any,
|
||||
):
|
||||
super().__init__()
|
||||
config = ParserConfig.new(config=config, owner=self, **settings)
|
||||
self.config = config
|
||||
|
||||
text = str(text)
|
||||
self.text = self.original_text = text
|
||||
|
||||
self.whitespace_re = self.build_whitespace_re(config.whitespace)
|
||||
self.nameguard = (
|
||||
config.nameguard
|
||||
if config.nameguard is not None
|
||||
else bool(self.whitespace_re) or bool(config.namechars)
|
||||
)
|
||||
self._namechar_set = set(config.namechars)
|
||||
|
||||
self._pos = 0
|
||||
self._len = 0
|
||||
self._linecount = 0
|
||||
self._lines: list[str] = []
|
||||
self._line_index: list[LineIndexInfo] = []
|
||||
self._line_cache: list[PosLine] = []
|
||||
self._comment_index: list[CommentInfo] = []
|
||||
|
||||
self._preprocess()
|
||||
self._postprocess()
|
||||
|
||||
@property
|
||||
def filename(self):
|
||||
return self.config.filename
|
||||
|
||||
@property
|
||||
def ignorecase(self):
|
||||
return self.config.ignorecase
|
||||
|
||||
@property
|
||||
def whitespace(self):
|
||||
return self.config.whitespace
|
||||
|
||||
@staticmethod
|
||||
def build_whitespace_re(whitespace):
|
||||
if type(whitespace) is UndefinedStr:
|
||||
return DEFAULT_WHITESPACE_RE
|
||||
if whitespace in {None, ''}:
|
||||
return None
|
||||
elif isinstance(whitespace, re.Pattern):
|
||||
return whitespace
|
||||
elif whitespace:
|
||||
return cached_re_compile(whitespace)
|
||||
else:
|
||||
return None
|
||||
|
||||
def _preprocess(self, *args, **kwargs):
|
||||
lines, index = self._preprocess_block(self.filename, self.text)
|
||||
self._lines = lines
|
||||
self._line_index = index
|
||||
self.text = self.join_block_lines(lines)
|
||||
|
||||
def _postprocess(self):
|
||||
cache, count = PosLine.build_line_cache(self._lines)
|
||||
self._line_cache = cache
|
||||
self._linecount = count
|
||||
self._len = len(self.text)
|
||||
|
||||
def _preprocess_block(self, name, block, **kwargs):
|
||||
lines = self.split_block_lines(block)
|
||||
index = LineIndexInfo.block_index(name, len(lines))
|
||||
return self.process_block(name, lines, index, **kwargs)
|
||||
|
||||
def split_block_lines(self, block):
|
||||
return block.splitlines(True)
|
||||
|
||||
def join_block_lines(self, lines):
|
||||
return ''.join(lines)
|
||||
|
||||
def process_block(self, name, lines, index, **kwargs):
|
||||
return lines, index
|
||||
|
||||
def include(self, lines, index, i, j, name, block, **kwargs):
|
||||
blines, bindex = self._preprocess_block(name, block, **kwargs)
|
||||
assert len(blines) == len(bindex)
|
||||
lines[i:j] = blines
|
||||
index[i:j] = bindex
|
||||
assert len(lines) == len(index)
|
||||
return j + len(blines) - 1
|
||||
|
||||
def include_file(self, source, name, lines, index, i, j):
|
||||
text, filename = self.get_include(source, name)
|
||||
return self.include(lines, index, i, j, filename, text)
|
||||
|
||||
def get_include(self, source, filename):
|
||||
source = Path(source).resolve()
|
||||
base = source.parent
|
||||
include = base / filename
|
||||
try:
|
||||
with include.open() as f:
|
||||
return f.read(), include
|
||||
except OSError as e:
|
||||
raise ParseError(f'include not found: {include}') from e
|
||||
|
||||
def replace_lines(self, i, j, name, block):
|
||||
lines = self.split_block_lines(self.text)
|
||||
index = list(self._line_index)
|
||||
|
||||
endline = self.include(lines, index, i, j, name, block)
|
||||
|
||||
self.text = self.join_block_lines(lines)
|
||||
self._line_index = index
|
||||
self._postprocess()
|
||||
|
||||
newtext = self.join_block_lines(lines[j + 1: endline + 2])
|
||||
return endline, newtext
|
||||
|
||||
@property
|
||||
def pos(self):
|
||||
return self._pos
|
||||
|
||||
@pos.setter
|
||||
def pos(self, p):
|
||||
self.goto(p)
|
||||
|
||||
@property
|
||||
def line(self):
|
||||
return self.posline()
|
||||
|
||||
@property
|
||||
def col(self):
|
||||
return self.poscol()
|
||||
|
||||
def posline(self, pos=None):
|
||||
if pos is None:
|
||||
pos = self._pos
|
||||
return self._line_cache[pos].line
|
||||
|
||||
def poscol(self, pos=None):
|
||||
if pos is None:
|
||||
pos = self._pos
|
||||
start = self._line_cache[pos].start
|
||||
return pos - start
|
||||
|
||||
def atend(self):
|
||||
return self._pos >= self._len
|
||||
|
||||
def ateol(self):
|
||||
return self.atend() or self.current in '\r\n'
|
||||
|
||||
@property
|
||||
def current(self):
|
||||
if self._pos >= self._len:
|
||||
return None
|
||||
return self.text[self._pos]
|
||||
|
||||
def at(self, p):
|
||||
if p >= self._len:
|
||||
return None
|
||||
return self.text[p]
|
||||
|
||||
def peek(self, n=1):
|
||||
return self.at(self._pos + n)
|
||||
|
||||
def next(self):
|
||||
if self.atend():
|
||||
return None
|
||||
c = self.text[self._pos]
|
||||
self._pos += 1
|
||||
return c
|
||||
|
||||
def goto(self, pos):
|
||||
self._pos = max(0, min(len(self.text), pos))
|
||||
|
||||
def move(self, n):
|
||||
self.goto(self.pos + n)
|
||||
|
||||
def comments(self, p, clear=False):
|
||||
if not self.config.comment_recovery or not self._comment_index:
|
||||
return CommentInfo([], [])
|
||||
|
||||
n = self.posline(p)
|
||||
if n >= len(self._comment_index):
|
||||
return CommentInfo([], [])
|
||||
|
||||
eolcmm = []
|
||||
if n < len(self._comment_index):
|
||||
eolcmm = self._comment_index[n].eol
|
||||
if clear:
|
||||
self._comment_index[n].eol = []
|
||||
|
||||
cmm = []
|
||||
while n >= 0 and self._comment_index[n].inline:
|
||||
cmm.insert(0, self._comment_index[n].inline)
|
||||
if clear:
|
||||
self._comment_index[n].inline = []
|
||||
n -= 1
|
||||
|
||||
return CommentInfo(cmm, eolcmm)
|
||||
|
||||
def _index_comments(self, comments, selector):
|
||||
if comments and self.config.comment_recovery:
|
||||
n = self.line
|
||||
extend_list(
|
||||
self._comment_index, n, default=CommentInfo.new_comment,
|
||||
)
|
||||
previous = selector(self._comment_index[n])
|
||||
if not contains_sublist(
|
||||
previous, comments,
|
||||
): # FIXME: will discard repeated comments
|
||||
previous.extend(comments)
|
||||
|
||||
def _eat_regex(self, regex):
|
||||
if not regex:
|
||||
return
|
||||
while self._matchre_fast(regex):
|
||||
pass
|
||||
|
||||
def _eat_regex_list(self, regex):
|
||||
if not regex:
|
||||
return []
|
||||
regex = cached_re_compile(regex)
|
||||
return list(takewhile(identity, map(self.matchre, repeat(regex))))
|
||||
|
||||
def eat_whitespace(self):
|
||||
return self._eat_regex(self.whitespace_re)
|
||||
|
||||
def eat_comments(self):
|
||||
comments = self._eat_regex_list(self.config.comments)
|
||||
self._index_comments(comments, lambda x: x.inline)
|
||||
|
||||
def eat_eol_comments(self):
|
||||
comments = self._eat_regex_list(self.config.eol_comments)
|
||||
self._index_comments(comments, lambda x: x.eol)
|
||||
|
||||
def next_token(self):
|
||||
p = None
|
||||
while self._pos != p:
|
||||
p = self._pos
|
||||
self.eat_eol_comments()
|
||||
self.eat_comments()
|
||||
self.eat_whitespace()
|
||||
|
||||
def skip_to(self, c):
|
||||
p = self._pos
|
||||
le = self._len
|
||||
while p < le and self.text[p] != c:
|
||||
p += 1
|
||||
self.goto(p)
|
||||
return self.pos
|
||||
|
||||
def skip_past(self, c):
|
||||
self.skip_to(c)
|
||||
self.next()
|
||||
return self.pos
|
||||
|
||||
def skip_to_eol(self):
|
||||
return self.skip_to('\n')
|
||||
|
||||
def scan_space(self):
|
||||
return (
|
||||
self.whitespace_re and self._scanre(self.whitespace_re) is not None
|
||||
)
|
||||
|
||||
def is_space(self):
|
||||
return self.scan_space()
|
||||
|
||||
def is_name_char(self, c):
|
||||
return c is not None and (c.isalnum() or c in self._namechar_set)
|
||||
|
||||
def match(self, token: str) -> str | None:
|
||||
if token is None:
|
||||
return self.atend()
|
||||
|
||||
p = self.pos
|
||||
if self.ignorecase:
|
||||
is_match = self.text[p: p + len(token)].lower() == token.lower()
|
||||
else:
|
||||
is_match = self.text[p: p + len(token)] == token
|
||||
|
||||
if not is_match:
|
||||
return None
|
||||
|
||||
self.move(len(token))
|
||||
partial_match = (
|
||||
self.nameguard
|
||||
and token
|
||||
and token[0].isalpha()
|
||||
and self.is_name_char(self.current)
|
||||
and all(self.is_name_char(t) for t in token)
|
||||
)
|
||||
if partial_match:
|
||||
self.goto(p)
|
||||
return None
|
||||
|
||||
return token
|
||||
|
||||
def _matchre_fast(self, pattern):
|
||||
if not (match := self._scanre(pattern)):
|
||||
return
|
||||
|
||||
self.move(len(match.group()))
|
||||
|
||||
def matchre(self, pattern):
|
||||
if not (match := self._scanre(pattern)):
|
||||
return None
|
||||
|
||||
matched = match.group()
|
||||
token = match_to_find(match)
|
||||
self.move(len(matched))
|
||||
return token
|
||||
|
||||
def _scanre(self, pattern):
|
||||
cre = cached_re_compile(pattern)
|
||||
return cre.match(self.text, self.pos)
|
||||
|
||||
@property
|
||||
def linecount(self):
|
||||
return self._linecount
|
||||
|
||||
def line_info(self, pos=None):
|
||||
if pos is None:
|
||||
pos = self._pos
|
||||
|
||||
# -2 to skip over sentinel
|
||||
pos = min(pos, len(self._line_cache) - 2)
|
||||
start, line, length = self._line_cache[pos]
|
||||
end = start + length
|
||||
col = pos - start
|
||||
|
||||
text = self.text[start:end]
|
||||
|
||||
# only required to support includes
|
||||
n = min(len(self._line_index) - 1, line)
|
||||
filename, line = self._line_index[n]
|
||||
|
||||
return LineInfo(filename, line, col, start, end, text)
|
||||
|
||||
def lookahead_pos(self):
|
||||
if self.atend():
|
||||
return ''
|
||||
info = self.line_info()
|
||||
return '~%d:%d' % (info.line + 1, info.col + 1)
|
||||
|
||||
def lookahead(self):
|
||||
if self.atend():
|
||||
return ''
|
||||
info = self.line_info()
|
||||
text = info.text[info.col: info.col + 1 + 80]
|
||||
text = self.split_block_lines(text)[0].rstrip()
|
||||
return f'{text}'
|
||||
|
||||
def get_line(self, n=None):
|
||||
if n is None:
|
||||
n = self.line
|
||||
return self._lines[n]
|
||||
|
||||
def get_lines(self, start=None, end=None):
|
||||
if start is None:
|
||||
start = 0
|
||||
if end is None:
|
||||
end = len(self._lines)
|
||||
return self._lines[start: end + 1]
|
||||
|
||||
def line_index(self, start=0, end=None):
|
||||
if end is None:
|
||||
end = len(self._line_index)
|
||||
return self._line_index[start: 1 + end]
|
||||
|
||||
def __repr__(self):
|
||||
return '%s@%d' % (type(self).__name__, self.pos)
|
||||
|
||||
def __json__(self, seen=None):
|
||||
return None
|
||||
@@ -0,0 +1,18 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from ..exceptions import CodegenError
|
||||
from .cgbase import ( # noqa: F401
|
||||
CodeGenerator,
|
||||
DelegatingRenderingFormatter,
|
||||
ModelRenderer,
|
||||
NullModelRenderer,
|
||||
)
|
||||
|
||||
|
||||
def codegen(model, target='python'):
|
||||
if target.lower() == 'python':
|
||||
from tatsu.codegen import python
|
||||
|
||||
return python.codegen(model)
|
||||
else:
|
||||
raise CodegenError(f'Unknown target language: {target}')
|
||||
@@ -0,0 +1,159 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from ..exceptions import CodegenError
|
||||
from ..objectmodel import Node
|
||||
from ..rendering import Renderer, RenderingFormatter, render
|
||||
|
||||
__all__ = [
|
||||
'DelegatingRenderingFormatter',
|
||||
'ModelRenderer',
|
||||
'NullModelRenderer',
|
||||
'CodeGenerator',
|
||||
]
|
||||
|
||||
|
||||
class DelegatingRenderingFormatter(RenderingFormatter):
|
||||
def __init__(self, delegate):
|
||||
assert hasattr(delegate, 'render')
|
||||
super().__init__()
|
||||
self.delegate = delegate
|
||||
|
||||
# override
|
||||
def render(self, item, join='', **fields):
|
||||
result = self.delegate.render(item, join=join, **fields)
|
||||
if result is None:
|
||||
result = super().render(item, join=join, **fields)
|
||||
return result
|
||||
|
||||
def convert_field(self, value, conversion):
|
||||
if isinstance(value, Node):
|
||||
return self.render(value)
|
||||
else:
|
||||
return super().convert_field(value, conversion)
|
||||
|
||||
|
||||
class ModelRenderer(Renderer):
|
||||
def __init__(self, codegen, node, template=None):
|
||||
super().__init__(template=template)
|
||||
self._codegen = codegen
|
||||
self._node = node
|
||||
|
||||
self.formatter = codegen.formatter
|
||||
|
||||
self.__postinit__()
|
||||
|
||||
def __postinit__(self):
|
||||
pass
|
||||
|
||||
def __getattr__(self, name):
|
||||
try:
|
||||
super().__getattr__(name)
|
||||
except AttributeError:
|
||||
if name.startswith('_'):
|
||||
raise
|
||||
return getattr(self.node, name)
|
||||
|
||||
@property
|
||||
def node(self):
|
||||
return self._node
|
||||
|
||||
@property
|
||||
def codegen(self):
|
||||
return self._codegen
|
||||
|
||||
@property
|
||||
def context(self):
|
||||
return self._codegen
|
||||
|
||||
def get_renderer(self, item):
|
||||
return self.codegen.get_renderer(item)
|
||||
|
||||
def render(self, **fields):
|
||||
template = fields.pop('template', None)
|
||||
if isinstance(self.node, Node):
|
||||
fields.update(
|
||||
{
|
||||
k: v
|
||||
for k, v in vars(self.node).items()
|
||||
if not k.startswith('_')
|
||||
},
|
||||
)
|
||||
else:
|
||||
fields.update(value=self.node)
|
||||
return super().render(template=template, **fields)
|
||||
|
||||
|
||||
class NullModelRenderer(ModelRenderer):
|
||||
"""A `ModelRenderer` that generates nothing."""
|
||||
|
||||
template = ''
|
||||
|
||||
|
||||
class CodeGenerator:
|
||||
"""
|
||||
A **CodeGenerator** is an abstract class that finds a
|
||||
``ModelRenderer`` class with the same name as each model's node and
|
||||
uses it to render the node.
|
||||
"""
|
||||
|
||||
def __init__(self, modules=None):
|
||||
self.formatter = DelegatingRenderingFormatter(self)
|
||||
self._renderers = {}
|
||||
if modules is not None:
|
||||
self._renderers = self._find_module_renderers(modules)
|
||||
|
||||
def _find_module_renderers(self, modules):
|
||||
result = {}
|
||||
|
||||
for module in modules:
|
||||
for name, dtype in vars(module).items():
|
||||
if not isinstance(dtype, type):
|
||||
continue
|
||||
if not issubclass(dtype, ModelRenderer):
|
||||
continue
|
||||
if dtype is not ModelRenderer:
|
||||
result[name] = dtype
|
||||
|
||||
return result
|
||||
|
||||
def _find_renderer_class(self, node):
|
||||
if not isinstance(node, Node):
|
||||
return None
|
||||
|
||||
node_class_name = node.__class__.__name__
|
||||
classes = [node.__class__]
|
||||
while classes:
|
||||
cls = classes.pop()
|
||||
|
||||
name = cls.__name__
|
||||
if name in self._renderers:
|
||||
renderer = self._renderers[name]
|
||||
self._renderers[node_class_name] = renderer
|
||||
return renderer
|
||||
|
||||
for base in cls.__bases__:
|
||||
if base not in classes:
|
||||
classes.append(base)
|
||||
|
||||
raise CodegenError(f'Renderer for {node_class_name} not found')
|
||||
|
||||
def get_renderer(self, item):
|
||||
if not isinstance(item, Node):
|
||||
return None
|
||||
|
||||
renderer_class = self._find_renderer_class(item)
|
||||
if renderer_class is None:
|
||||
raise CodegenError(
|
||||
f'Renderer not found for {type(item).__name__}',
|
||||
)
|
||||
try:
|
||||
assert issubclass(renderer_class, ModelRenderer)
|
||||
return renderer_class(self, item)
|
||||
except Exception as e:
|
||||
raise type(e)(str(e), renderer_class.__name__) from e
|
||||
|
||||
def render(self, item, join='', **fields):
|
||||
renderer = self.get_renderer(item)
|
||||
if renderer is None:
|
||||
return render(item, join=join, **fields)
|
||||
return str(renderer.render(**fields))
|
||||
@@ -0,0 +1,233 @@
|
||||
import inspect
|
||||
from collections import namedtuple
|
||||
from datetime import datetime
|
||||
|
||||
from tatsu.util import compress_seq, indent, re, safe_name
|
||||
|
||||
from ..codegen.cgbase import CodeGenerator, ModelRenderer
|
||||
from ..exceptions import CodegenError
|
||||
from ..objectmodel import BASE_CLASS_TOKEN, Node
|
||||
from ..rendering import Renderer
|
||||
|
||||
NODE_NAME_PATTERN = r'(?!\d)\w+(' + rf'{BASE_CLASS_TOKEN}' + r'(?!\d)\w+)*'
|
||||
|
||||
|
||||
TypeSpec = namedtuple('TypeSpec', ['class_name', 'base'])
|
||||
|
||||
DEFAULT_BASE_TYPE = """
|
||||
@dataclass(eq=False)
|
||||
class ModelBase(Node):
|
||||
pass
|
||||
"""
|
||||
|
||||
|
||||
def codegen(model, base_type=None):
|
||||
return ObjectModelCodeGenerator().render(model, base_type=base_type)
|
||||
|
||||
|
||||
def _get_node_class_name(rule):
|
||||
if not rule.params:
|
||||
return None
|
||||
|
||||
typespec = rule.params[0]
|
||||
if not isinstance(typespec, str):
|
||||
return None
|
||||
if not re.match(NODE_NAME_PATTERN, typespec):
|
||||
return None
|
||||
if not typespec[0].isupper():
|
||||
return None
|
||||
return typespec
|
||||
|
||||
|
||||
def _typespec(rule):
|
||||
if not _get_node_class_name(rule):
|
||||
return []
|
||||
|
||||
spec = rule.params[0].split(BASE_CLASS_TOKEN)
|
||||
class_names = [safe_name(n) for n in spec] + ['ModelBase']
|
||||
|
||||
typespec = []
|
||||
for i, class_name in enumerate(class_names[:-1]):
|
||||
base = class_names[i + 1]
|
||||
typespec.append(TypeSpec(class_name, base))
|
||||
|
||||
return typespec
|
||||
|
||||
|
||||
def _get_full_name(cls):
|
||||
if not inspect.isclass(cls):
|
||||
raise CodegenError('Base type has to be a class')
|
||||
module = inspect.getmodule(cls)
|
||||
if not module:
|
||||
raise CodegenError('Base type has to be inside a module')
|
||||
modulename = module.__name__
|
||||
|
||||
name = cls.__qualname__
|
||||
|
||||
# Try to reference the class
|
||||
try:
|
||||
idents = name.split('.')
|
||||
cls_ = getattr(module, idents[0])
|
||||
for ident in idents[1:]:
|
||||
cls_ = getattr(cls_, ident)
|
||||
|
||||
assert cls_ == cls
|
||||
except AttributeError as e:
|
||||
raise CodegenError(
|
||||
"Couldn't find base type, it has to be importable",
|
||||
) from e
|
||||
|
||||
return modulename, name
|
||||
|
||||
|
||||
class BaseTypeRenderer(Renderer):
|
||||
def __init__(self, base_type):
|
||||
super().__init__()
|
||||
self.base_type = base_type
|
||||
|
||||
def render_fields(self, fields):
|
||||
module, name = _get_full_name(self.base_type)
|
||||
if '.' in name:
|
||||
lookup = f'\nModelBase = {name}'
|
||||
name = name.split('.')[0]
|
||||
else:
|
||||
lookup = ' as ModelBase'
|
||||
|
||||
fields.update(module=module, name=name, lookup=lookup)
|
||||
|
||||
template = """
|
||||
from {module} import {name}{lookup}\
|
||||
"""
|
||||
|
||||
|
||||
class BaseClassRenderer(Renderer):
|
||||
def __init__(self, spec):
|
||||
super().__init__()
|
||||
self.class_name = spec.class_name
|
||||
self.base = spec.base
|
||||
|
||||
template = """
|
||||
@dataclass(eq=False)
|
||||
class {class_name}({base}):
|
||||
pass\
|
||||
"""
|
||||
|
||||
|
||||
class ObjectModelCodeGenerator(CodeGenerator):
|
||||
def _find_renderer_class(self, node):
|
||||
if not isinstance(node, Node):
|
||||
return None
|
||||
|
||||
name = node.__class__.__name__
|
||||
renderer = globals().get(name)
|
||||
if not renderer or not issubclass(renderer, ModelRenderer):
|
||||
raise CodegenError(f'Renderer for {name} not found')
|
||||
return renderer
|
||||
|
||||
|
||||
class Rule(ModelRenderer):
|
||||
def render_fields(self, fields):
|
||||
defs = [safe_name(d) for d, _ in compress_seq(self.defines())]
|
||||
defs = sorted(set(defs))
|
||||
spec = fields['spec']
|
||||
|
||||
kwargs = '\n'.join(f'{d}: Any = None' for d in defs)
|
||||
kwargs = indent(kwargs) if kwargs else indent('pass')
|
||||
|
||||
fields.update(
|
||||
class_name=spec.class_name, base=spec.base, kwargs=kwargs,
|
||||
)
|
||||
|
||||
template = """
|
||||
@dataclass(eq=False)
|
||||
class {class_name}({base}):
|
||||
{kwargs}\
|
||||
"""
|
||||
|
||||
|
||||
class Grammar(ModelRenderer):
|
||||
def render_fields(self, fields):
|
||||
node_class_names = set()
|
||||
|
||||
bases = []
|
||||
model_rules = []
|
||||
for rule in self.node.rules:
|
||||
specs = _typespec(rule)
|
||||
if not specs:
|
||||
continue
|
||||
|
||||
node_spec = specs[0]
|
||||
base_spec = reversed(specs[1:])
|
||||
|
||||
if node_spec.class_name not in node_class_names:
|
||||
model_rules.append((rule, node_spec))
|
||||
|
||||
bases.extend(
|
||||
base
|
||||
for base in base_spec
|
||||
if base.class_name not in node_class_names
|
||||
)
|
||||
|
||||
node_class_names.update(s.class_name for s in specs)
|
||||
|
||||
base_class_declarations = [
|
||||
BaseClassRenderer(spec).render() for spec in bases
|
||||
]
|
||||
|
||||
model_class_declarations = [
|
||||
self.get_renderer(rule).render(spec=spec)
|
||||
for rule, spec in model_rules
|
||||
]
|
||||
|
||||
base_class_declarations = '\n\n\n'.join(base_class_declarations)
|
||||
if base_class_declarations:
|
||||
base_class_declarations += '\n\n'
|
||||
model_class_declarations = '\n\n\n'.join(model_class_declarations)
|
||||
|
||||
version = datetime.now().strftime('%Y.%m.%d.%H')
|
||||
|
||||
base_type = fields['base_type']
|
||||
|
||||
fields.update(
|
||||
name=self.node.name,
|
||||
base_class_declarations=base_class_declarations,
|
||||
model_class_declarations=model_class_declarations,
|
||||
version=version,
|
||||
base_type=BaseTypeRenderer(base_type).render()
|
||||
if base_type
|
||||
else DEFAULT_BASE_TYPE,
|
||||
)
|
||||
|
||||
template = """\
|
||||
#!/usr/bin/env python
|
||||
|
||||
# CAVEAT UTILITOR
|
||||
#
|
||||
# This file was automatically generated by TatSu.
|
||||
#
|
||||
# https://pypi.python.org/pypi/tatsu/
|
||||
#
|
||||
# Any changes you make to it will be overwritten the next time
|
||||
# the file is generated.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from dataclasses import dataclass
|
||||
|
||||
from tatsu.objectmodel import Node
|
||||
from tatsu.semantics import ModelBuilderSemantics
|
||||
|
||||
{base_type}
|
||||
|
||||
class {name}ModelBuilderSemantics(ModelBuilderSemantics):
|
||||
def __init__(self, context=None, types=None):
|
||||
types = [
|
||||
t for t in globals().values()
|
||||
if type(t) is type and issubclass(t, ModelBase)
|
||||
] + (types or [])
|
||||
super().__init__(context=context, types=types)
|
||||
|
||||
{base_class_declarations}
|
||||
{model_class_declarations}
|
||||
"""
|
||||
@@ -0,0 +1,594 @@
|
||||
"""
|
||||
Python code generation for models defined with tatsu.model
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import textwrap
|
||||
|
||||
from .. import grammars
|
||||
from ..collections import OrderedSet as oset
|
||||
from ..exceptions import CodegenError
|
||||
from ..objectmodel import BASE_CLASS_TOKEN, Node
|
||||
from ..util import compress_seq, indent, safe_name, timestamp, trim
|
||||
from .cgbase import CodeGenerator, ModelRenderer
|
||||
|
||||
|
||||
class PythonCodeGenerator(CodeGenerator):
|
||||
def _find_renderer_class(self, node):
|
||||
if not isinstance(node, Node):
|
||||
return None
|
||||
|
||||
name = node.__class__.__name__
|
||||
renderer = globals().get(name)
|
||||
if not renderer or not issubclass(renderer, Base):
|
||||
raise CodegenError(f'Renderer for {name} not found')
|
||||
return renderer
|
||||
|
||||
|
||||
def codegen(model):
|
||||
return PythonCodeGenerator().render(model)
|
||||
|
||||
|
||||
class Base(ModelRenderer):
|
||||
def defines(self):
|
||||
return self.node.defines()
|
||||
|
||||
def make_defines_declaration(self):
|
||||
defines = compress_seq(self.defines())
|
||||
ldefs = oset(safe_name(d) for d, value in defines if value)
|
||||
sdefs = oset(
|
||||
safe_name(d)
|
||||
for d, value in defines
|
||||
if not value and d not in ldefs
|
||||
)
|
||||
|
||||
if not (sdefs or ldefs):
|
||||
return ''
|
||||
else:
|
||||
sdefs = '[{}]'.format(', '.join(sorted(repr(d) for d in sdefs)))
|
||||
ldefs = '[{}]'.format(', '.join(sorted(repr(d) for d in ldefs)))
|
||||
if not ldefs:
|
||||
return f'\n\n self._define({sdefs}, {ldefs})'
|
||||
else:
|
||||
return '\n' + trim(self.define_template % (sdefs, ldefs))
|
||||
|
||||
define_template = """\
|
||||
self._define(
|
||||
%s,
|
||||
%s,
|
||||
)\
|
||||
"""
|
||||
|
||||
|
||||
class Void(Base):
|
||||
template = 'self._void()'
|
||||
|
||||
|
||||
class Any(Base):
|
||||
template = 'self._any()'
|
||||
|
||||
|
||||
class Fail(Base):
|
||||
template = 'self._fail()'
|
||||
|
||||
|
||||
class Comment(Base):
|
||||
def render_fields(self, fields):
|
||||
lines = '\n'.join(
|
||||
f'# {c!s}' for c in self.node.comment.splitlines()
|
||||
)
|
||||
fields.update(lines=lines)
|
||||
|
||||
template = '\n{lines}\n'
|
||||
|
||||
|
||||
class EOLComment(Comment):
|
||||
pass
|
||||
|
||||
|
||||
class EOF(Base):
|
||||
template = 'self._check_eof()'
|
||||
|
||||
|
||||
class _Decorator(Base):
|
||||
template = '{exp}'
|
||||
|
||||
|
||||
class Group(_Decorator):
|
||||
template = """\
|
||||
with self._group():
|
||||
{exp:1::}\
|
||||
"""
|
||||
|
||||
|
||||
class Token(Base):
|
||||
def render_fields(self, fields):
|
||||
fields.update(token=repr(self.node.token))
|
||||
|
||||
template = 'self._token({token})'
|
||||
|
||||
|
||||
class Constant(Base):
|
||||
def render_fields(self, fields):
|
||||
fields.update(constant=repr(self.node.literal))
|
||||
|
||||
template = 'self._constant({constant})'
|
||||
|
||||
|
||||
class Alert(Base):
|
||||
def render_fields(self, fields):
|
||||
fields.update(literal=repr(self.node.literal), level=self.node.level)
|
||||
|
||||
template = 'self._alert({literal}, {level})'
|
||||
|
||||
|
||||
class Pattern(Base):
|
||||
def render_fields(self, fields):
|
||||
raw_repr = repr(self.node.pattern)
|
||||
fields.update(pattern=raw_repr)
|
||||
|
||||
template = 'self._pattern({pattern})'
|
||||
|
||||
|
||||
class Lookahead(_Decorator):
|
||||
template = """\
|
||||
with self._if():
|
||||
{exp:1::}\
|
||||
"""
|
||||
|
||||
|
||||
class NegativeLookahead(_Decorator):
|
||||
template = """\
|
||||
with self._ifnot():
|
||||
{exp:1::}\
|
||||
"""
|
||||
|
||||
|
||||
class Sequence(Base):
|
||||
def render_fields(self, fields):
|
||||
fields.update(seq='\n'.join(self.rend(s) for s in self.node.sequence))
|
||||
defines = self.make_defines_declaration()
|
||||
fields.update(defines=defines)
|
||||
|
||||
template = """{seq}
|
||||
{defines}\
|
||||
"""
|
||||
|
||||
|
||||
class Choice(Base):
|
||||
def render_fields(self, fields):
|
||||
firstset = self.node.lookahead_str()
|
||||
if firstset:
|
||||
msglines = textwrap.wrap(firstset, width=40)
|
||||
error = ['expecting one of: ', *msglines]
|
||||
else:
|
||||
error = ['no available options']
|
||||
error = [repr(e) for e in error]
|
||||
fields.update(n=self.counter(), error=error)
|
||||
fields.update(
|
||||
options='\n'.join(self.rend(o) for o in self.node.options),
|
||||
)
|
||||
|
||||
def render(self, **fields):
|
||||
if len(self.node.options) == 1:
|
||||
return self.rend(self.options[0], **fields)
|
||||
else:
|
||||
return super().render(**fields)
|
||||
|
||||
template = """\
|
||||
with self._choice():
|
||||
{options:1::}
|
||||
self._error(
|
||||
{error:2:\\n:} # noqa: COM812
|
||||
)\
|
||||
"""
|
||||
|
||||
|
||||
class Option(_Decorator):
|
||||
def render_fields(self, fields):
|
||||
defines = self.make_defines_declaration()
|
||||
fields.update(defines=defines)
|
||||
|
||||
template = """\
|
||||
with self._option():
|
||||
{exp:1::}\
|
||||
"""
|
||||
|
||||
|
||||
class Closure(_Decorator):
|
||||
def render_fields(self, fields):
|
||||
fields.update(n=self.counter())
|
||||
|
||||
def render(self, **fields):
|
||||
if () in self.node.exp.lookahead():
|
||||
raise CodegenError(f'{self.node} may repeat empty sequence')
|
||||
return '\n' + super().render(**fields)
|
||||
|
||||
template = """\
|
||||
def block{n}():
|
||||
{exp:1::}
|
||||
self._closure(block{n})\
|
||||
"""
|
||||
|
||||
|
||||
class PositiveClosure(Closure):
|
||||
template = """\
|
||||
def block{n}():
|
||||
{exp:1::}
|
||||
self._positive_closure(block{n})\
|
||||
"""
|
||||
|
||||
|
||||
class Join(_Decorator):
|
||||
def render_fields(self, fields):
|
||||
fields.update(n=self.counter())
|
||||
|
||||
def render(self, **fields):
|
||||
if () in self.node.exp.lookahead():
|
||||
raise CodegenError('may repeat empty sequence')
|
||||
return '\n' + super().render(**fields)
|
||||
|
||||
template = """\
|
||||
def sep{n}():
|
||||
{sep:1::}
|
||||
|
||||
def block{n}():
|
||||
{exp:1::}
|
||||
self._join(block{n}, sep{n})\
|
||||
"""
|
||||
|
||||
|
||||
class PositiveJoin(Join):
|
||||
template = """\
|
||||
def sep{n}():
|
||||
{sep:1::}
|
||||
|
||||
def block{n}():
|
||||
{exp:1::}
|
||||
self._positive_join(block{n}, sep{n})\
|
||||
"""
|
||||
|
||||
|
||||
class Gather(Join):
|
||||
template = """\
|
||||
def sep{n}():
|
||||
{sep:1::}
|
||||
|
||||
def block{n}():
|
||||
{exp:1::}
|
||||
self._gather(block{n}, sep{n})\
|
||||
"""
|
||||
|
||||
|
||||
class PositiveGather(Join):
|
||||
template = """\
|
||||
def sep{n}():
|
||||
{sep:1::}
|
||||
|
||||
def block{n}():
|
||||
{exp:1::}
|
||||
self._positive_gather(block{n}, sep{n})\
|
||||
"""
|
||||
|
||||
|
||||
class LeftJoin(PositiveJoin):
|
||||
template = """\
|
||||
def sep{n}():
|
||||
{sep:1::}
|
||||
|
||||
def block{n}():
|
||||
{exp:1::}
|
||||
self._left_join(block{n}, sep{n})\
|
||||
"""
|
||||
|
||||
|
||||
class RightJoin(PositiveJoin):
|
||||
template = """\
|
||||
def sep{n}():
|
||||
{sep:1::}
|
||||
|
||||
def block{n}():
|
||||
{exp:1::}
|
||||
self._right_join(block{n}, sep{n})\
|
||||
"""
|
||||
|
||||
|
||||
class EmptyClosure(Base):
|
||||
template = 'self._empty_closure()'
|
||||
|
||||
|
||||
class SkipTo(Closure):
|
||||
template = """\
|
||||
def block{n}():
|
||||
{exp:1::}
|
||||
self._skip_to(block{n})\
|
||||
"""
|
||||
|
||||
|
||||
class Optional(_Decorator):
|
||||
template = """\
|
||||
with self._optional():
|
||||
{exp:1::}\
|
||||
"""
|
||||
|
||||
|
||||
class Cut(Base):
|
||||
template = 'self._cut()'
|
||||
|
||||
|
||||
class Named(_Decorator):
|
||||
def __str__(self):
|
||||
return f'{self.name}:{self.rend(self.exp)}'
|
||||
|
||||
def render_fields(self, fields):
|
||||
fields.update(n=self.counter(), name=safe_name(self.node.name))
|
||||
|
||||
template = """
|
||||
{exp}
|
||||
self.name_last_node('{name}')\
|
||||
"""
|
||||
|
||||
|
||||
class NamedList(Named):
|
||||
template = """
|
||||
{exp}
|
||||
self.add_last_node_to_name('{name}')\
|
||||
"""
|
||||
|
||||
|
||||
class Override(Named):
|
||||
pass
|
||||
|
||||
|
||||
class OverrideList(NamedList):
|
||||
pass
|
||||
|
||||
|
||||
class Special(Base):
|
||||
pass
|
||||
|
||||
|
||||
class RuleRef(Base):
|
||||
template = 'self._{name}_()'
|
||||
|
||||
|
||||
class RuleInclude(_Decorator):
|
||||
def render_fields(self, fields):
|
||||
super().render_fields(fields)
|
||||
fields.update(exp=self.rend(self.node.rule.exp))
|
||||
|
||||
template = """
|
||||
{exp}
|
||||
"""
|
||||
|
||||
|
||||
class Rule(_Decorator):
|
||||
@staticmethod
|
||||
def param_repr(p):
|
||||
if isinstance(p, int | float):
|
||||
return str(p)
|
||||
else:
|
||||
return repr(p.split(BASE_CLASS_TOKEN)[0])
|
||||
|
||||
def render(self, **fields):
|
||||
try:
|
||||
return super().render(**fields)
|
||||
except CodegenError as e:
|
||||
raise CodegenError(f'{self.node.name}={e}') from e
|
||||
|
||||
def render_fields(self, fields):
|
||||
self.reset_counter()
|
||||
|
||||
params = kwparams = ''
|
||||
if self.node.params:
|
||||
params = ', '.join(
|
||||
self.param_repr(self.rend(p)) for p in self.node.params
|
||||
)
|
||||
if self.node.kwparams:
|
||||
kwparams = ', '.join(
|
||||
f'{k}={self.param_repr(self.rend(v))}'
|
||||
for k, v in self.kwparams.items()
|
||||
)
|
||||
|
||||
if params and kwparams:
|
||||
params = params + ', ' + kwparams
|
||||
elif kwparams:
|
||||
params = kwparams
|
||||
|
||||
fields.update(params=params)
|
||||
|
||||
sdefines = ''
|
||||
if not isinstance(self.node, grammars.Choice):
|
||||
sdefines = self.make_defines_declaration()
|
||||
fields.update(defines=sdefines)
|
||||
leftrec = self.node.is_leftrec
|
||||
fields.update(leftrec='\n@leftrec' if leftrec else '')
|
||||
fields.update(
|
||||
nomemo='\n@nomemo'
|
||||
if not self.node.is_memoizable and not leftrec
|
||||
else '',
|
||||
)
|
||||
fields.update(isname='\n@isname' if self.node.is_name else '')
|
||||
|
||||
template = """
|
||||
@tatsumasu({params})\
|
||||
{leftrec}\
|
||||
{nomemo}\
|
||||
{isname}
|
||||
def _{name}_(self):
|
||||
{exp:1::}
|
||||
"""
|
||||
|
||||
define_template = """\
|
||||
self._define(
|
||||
%s,
|
||||
%s
|
||||
)\
|
||||
"""
|
||||
|
||||
|
||||
class BasedRule(Rule):
|
||||
def defines(self):
|
||||
return self.rhs.defines()
|
||||
|
||||
def render_fields(self, fields):
|
||||
super().render_fields(fields)
|
||||
fields.update(exp=self.rhs)
|
||||
|
||||
|
||||
class Grammar(Base):
|
||||
def render_fields(self, fields):
|
||||
abstract_template = trim(self.abstract_rule_template)
|
||||
abstract_rules = [
|
||||
abstract_template.format(name=safe_name(rule.name))
|
||||
for rule in self.node.rules
|
||||
]
|
||||
abstract_rules = indent('\n'.join(abstract_rules))
|
||||
|
||||
whitespace = self.node.config.whitespace
|
||||
if not whitespace:
|
||||
whitespace = 'None'
|
||||
elif isinstance(whitespace, re.Pattern):
|
||||
whitespace = repr(whitespace)
|
||||
else:
|
||||
whitespace = f're.compile(r"{whitespace}")'
|
||||
|
||||
if self.node.config.nameguard is not None:
|
||||
nameguard = repr(self.node.config.nameguard)
|
||||
else:
|
||||
nameguard = 'None'
|
||||
|
||||
ignorecase = self.node.config.ignorecase
|
||||
left_recursion = self.node.config.left_recursion
|
||||
parseinfo = self.node.config.parseinfo
|
||||
namechars = repr(self.node.config.namechars or '')
|
||||
comments = repr(self.node.config.comments)
|
||||
eol_comments = repr(self.node.config.eol_comments)
|
||||
|
||||
rules = '\n'.join(
|
||||
[self.get_renderer(rule).render() for rule in self.node.rules],
|
||||
)
|
||||
|
||||
version = str(tuple(int(n) for n in str(timestamp()).split('.')))
|
||||
|
||||
keywords = [str(k) for k in self.keywords if k is not None]
|
||||
keywords = '\n'.join(f' {k!r},' for k in keywords)
|
||||
if keywords:
|
||||
keywords = f'(\n{keywords}\n)'
|
||||
|
||||
fields.update(
|
||||
rules=indent(rules),
|
||||
start=self.node.rules[0].name,
|
||||
abstract_rules=abstract_rules,
|
||||
version=version,
|
||||
whitespace=whitespace,
|
||||
nameguard=nameguard,
|
||||
ignorecase=ignorecase,
|
||||
left_recursion=left_recursion,
|
||||
parseinfo=parseinfo,
|
||||
keywords=keywords,
|
||||
namechars=namechars,
|
||||
comments=comments,
|
||||
eol_comments=eol_comments,
|
||||
)
|
||||
|
||||
abstract_rule_template = """
|
||||
def {name}(self, ast):
|
||||
return ast
|
||||
"""
|
||||
|
||||
template = """\
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# WARNING: CAVEAT UTILITOR
|
||||
#
|
||||
# This file was automatically generated by TatSu.
|
||||
#
|
||||
# https://pypi.python.org/pypi/tatsu/
|
||||
#
|
||||
# Any changes you make to it will be overwritten the next time
|
||||
# the file is generated.
|
||||
|
||||
# ruff: noqa: C405, I001, F401, SIM117
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from tatsu.buffering import Buffer
|
||||
from tatsu.parsing import Parser
|
||||
from tatsu.parsing import tatsumasu
|
||||
from tatsu.parsing import leftrec, nomemo, isname
|
||||
from tatsu.infos import ParserConfig
|
||||
from tatsu.util import re, generic_main
|
||||
|
||||
|
||||
KEYWORDS: set[str] = set({keywords})
|
||||
|
||||
|
||||
class {name}Buffer(Buffer):
|
||||
def __init__(self, text, /, config: ParserConfig | None = None, **settings):
|
||||
config = ParserConfig.new(
|
||||
config,
|
||||
owner=self,
|
||||
whitespace={whitespace},
|
||||
nameguard={nameguard},
|
||||
ignorecase={ignorecase},
|
||||
namechars={namechars},
|
||||
parseinfo={parseinfo},
|
||||
comments={comments},
|
||||
eol_comments={eol_comments},
|
||||
keywords=KEYWORDS,
|
||||
start={start!r},
|
||||
)
|
||||
config = config.replace(**settings)
|
||||
super().__init__(text, config=config)
|
||||
|
||||
|
||||
class {name}Parser(Parser):
|
||||
def __init__(self, /, config: ParserConfig | None = None, **settings):
|
||||
config = ParserConfig.new(
|
||||
config,
|
||||
owner=self,
|
||||
whitespace={whitespace},
|
||||
nameguard={nameguard},
|
||||
ignorecase={ignorecase},
|
||||
namechars={namechars},
|
||||
parseinfo={parseinfo},
|
||||
comments={comments},
|
||||
eol_comments={eol_comments},
|
||||
left_recursion={left_recursion},
|
||||
keywords=KEYWORDS,
|
||||
start={start!r},
|
||||
)
|
||||
config = config.replace(**settings)
|
||||
super().__init__(config=config)
|
||||
|
||||
{rules}
|
||||
|
||||
|
||||
class {name}Semantics:
|
||||
{abstract_rules}
|
||||
|
||||
|
||||
def main(filename, **kwargs):
|
||||
if not filename or filename == '-':
|
||||
text = sys.stdin.read()
|
||||
else:
|
||||
text = Path(filename).read_text()
|
||||
parser = {name}Parser()
|
||||
return parser.parse(
|
||||
text,
|
||||
filename=filename,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
import json
|
||||
from tatsu.util import asjson
|
||||
|
||||
ast = generic_main(main, {name}Parser, name='{name}')
|
||||
data = asjson(ast)
|
||||
print(json.dumps(data, indent=2))
|
||||
"""
|
||||
@@ -0,0 +1,2 @@
|
||||
from .orderedset import OrderedSet # noqa: F401
|
||||
from .tail import Tail # noqa: F401
|
||||
@@ -0,0 +1,128 @@
|
||||
# NOTE: from https://github.com/LuminosoInsight/ordered-set/blob/master/ordered_set.py
|
||||
import itertools
|
||||
from collections.abc import (
|
||||
Iterable,
|
||||
Iterator,
|
||||
Mapping,
|
||||
MutableSequence,
|
||||
MutableSet,
|
||||
Sequence,
|
||||
)
|
||||
from typing import Any, TypeVar
|
||||
|
||||
T = TypeVar('T')
|
||||
|
||||
|
||||
class OrderedSet(MutableSet[T], Sequence[T]):
|
||||
def __init__(self, iterable: Iterable[T] | None = None):
|
||||
if iterable is not None:
|
||||
self._map = dict.fromkeys(iterable)
|
||||
else:
|
||||
self._map = {}
|
||||
self._list_cache: Sequence[T] | None = None
|
||||
|
||||
def __hash__(self):
|
||||
return self._map.__hash__()
|
||||
|
||||
def __len__(self):
|
||||
return len(self._map)
|
||||
|
||||
def __getitem__(self, i):
|
||||
if self._list_cache is None:
|
||||
self._list_cache = list(self._map.keys())
|
||||
return self._list_cache[i]
|
||||
|
||||
def copy(self) -> 'OrderedSet[T]':
|
||||
return self.__class__(self)
|
||||
|
||||
def __getstate__(self):
|
||||
if len(self) == 0:
|
||||
return (None,)
|
||||
else:
|
||||
return list(self)
|
||||
|
||||
def __setstate__(self, state):
|
||||
if state == (None,):
|
||||
self.__init__([])
|
||||
else:
|
||||
self.__init__(state)
|
||||
|
||||
def __contains__(self, key: Any) -> bool:
|
||||
return key in self._map
|
||||
|
||||
def add(self, value: T): # pylint: disable=W0221
|
||||
self._map[value] = len(self._map)
|
||||
self._list_cache = None
|
||||
|
||||
def update(self, sequence: Iterable[T]):
|
||||
self._map.update(dict.fromkeys(sequence))
|
||||
self._list_cache = None
|
||||
|
||||
def pop(self) -> T:
|
||||
key = next(iter(self._map.keys()))
|
||||
self._map.pop(key)
|
||||
self._list_cache = None
|
||||
return key
|
||||
|
||||
def discard(self, value: T): # pylint: disable=W0221
|
||||
self._map.pop(value, None)
|
||||
self._list_cache = None
|
||||
|
||||
def clear(self):
|
||||
self._map = {}
|
||||
self._list_cache = None
|
||||
|
||||
def __iter__(self) -> Iterator[T]:
|
||||
return iter(self._map.keys())
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return '{{{}}}'.format(', '.join(repr(e) for e in self))
|
||||
|
||||
def __eq__(self, other: Any) -> bool:
|
||||
return all(item in other for item in self)
|
||||
|
||||
def union(self, *other: Iterable[T]) -> 'OrderedSet[T]':
|
||||
# do not split `str`
|
||||
outer = tuple(
|
||||
[o] if not isinstance(o, set | Mapping | MutableSequence) else o
|
||||
for o in other
|
||||
)
|
||||
inner = itertools.chain([self], *outer)
|
||||
items = itertools.chain.from_iterable(inner)
|
||||
return type(self)(itertools.chain(items))
|
||||
|
||||
def __and__(self, other: Iterable[Iterable[T]]) -> 'OrderedSet[T]':
|
||||
return self.intersection(other)
|
||||
|
||||
def intersection(self, *other: Iterable[Iterable[T]]) -> 'OrderedSet[T]':
|
||||
common = set.intersection(*other) # type: ignore[var-annotated, arg-type]
|
||||
items = (item for item in self if item in common)
|
||||
return type(self)(items)
|
||||
|
||||
def difference(self, *other: Iterable[T]) -> 'OrderedSet[T]':
|
||||
other = set.union(*other) # type: ignore[assignment, arg-type]
|
||||
items = (item for item in self if item not in other)
|
||||
return type(self)(items)
|
||||
|
||||
def issubset(self, other: set[T]) -> bool:
|
||||
return all(item in other for item in self)
|
||||
|
||||
def issuperset(self, other: set[T]) -> bool:
|
||||
if len(self) < len(other): # Fast check for obvious cases
|
||||
return False
|
||||
return all(item in self for item in other)
|
||||
|
||||
def symmetric_difference(self, other: set[T]) -> 'OrderedSet[T]':
|
||||
cls = type(self)
|
||||
diff1 = cls(self).difference(other)
|
||||
diff2 = cls(other).difference(self)
|
||||
return cls().union(diff1, diff2)
|
||||
|
||||
def difference_update(self, *other: Iterable[T]):
|
||||
self._map = dict.fromkeys(self.difference(*other))
|
||||
|
||||
def intersection_update(self, *other: Iterable[Iterable[T]]):
|
||||
self._map = dict.fromkeys(self.intersection(*other))
|
||||
|
||||
def symmetric_difference_update(self, *other: Iterable[T]):
|
||||
self._map = dict.fromkeys(self.difference(*other))
|
||||
@@ -0,0 +1,37 @@
|
||||
from collections import deque
|
||||
from collections.abc import MutableSequence
|
||||
|
||||
|
||||
class Tail(MutableSequence):
|
||||
def __init__(self, maxlen):
|
||||
self._tail = deque(maxlen=maxlen)
|
||||
self._start = 0
|
||||
|
||||
@property
|
||||
def start(self):
|
||||
return self._start
|
||||
|
||||
def tailfull(self):
|
||||
return len(self._tail) == self._tail.maxlen
|
||||
|
||||
def insert(self, index, value):
|
||||
if self.tailfull():
|
||||
self._tail.popleft()
|
||||
self._start += 1
|
||||
self._tail.insert(index - self._start, value)
|
||||
|
||||
def flush(self):
|
||||
self._start += len(self._tail)
|
||||
self._tail = deque(maxlen=self._tail.maxlen)
|
||||
|
||||
def __getitem__(self, index):
|
||||
return self._tail[index - self._start]
|
||||
|
||||
def __setitem__(self, index, value):
|
||||
self._tail[index - self._start] = value
|
||||
|
||||
def __delitem__(self, index):
|
||||
del self._tail[index - self._start]
|
||||
|
||||
def __len__(self):
|
||||
return len(self._tail) + self._start
|
||||
@@ -0,0 +1,56 @@
|
||||
# ruff: noqa: PLW0603
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
class _BF:
|
||||
BLACK = ''
|
||||
BLUE = ''
|
||||
CYAN = ''
|
||||
GREEN = ''
|
||||
MAGENTA = ''
|
||||
RED = ''
|
||||
RESET = ''
|
||||
WHITE = ''
|
||||
YELLOW = ''
|
||||
LIGHTBLACK_EX = ''
|
||||
LIGHTRED_EX = ''
|
||||
LIGHTGREEN_EX = ''
|
||||
LIGHTYELLOW_EX = ''
|
||||
LIGHTBLUE_EX = ''
|
||||
LIGHTMAGENTA_EX = ''
|
||||
LIGHTCYAN_EX = ''
|
||||
LIGHTWHITE_EX = ''
|
||||
|
||||
|
||||
class _Fore(_BF):
|
||||
pass
|
||||
|
||||
|
||||
class _Back(_BF):
|
||||
pass
|
||||
|
||||
|
||||
class _Style:
|
||||
BRIGHT = ''
|
||||
DIM = ''
|
||||
NORMAL = ''
|
||||
RESET_ALL = ''
|
||||
|
||||
|
||||
Fore = _Fore
|
||||
Back = _Back
|
||||
Style = _Style
|
||||
|
||||
|
||||
def init():
|
||||
try:
|
||||
import colorama
|
||||
|
||||
colorama.init()
|
||||
except ImportError:
|
||||
return
|
||||
|
||||
global Fore, Back, Style # pylint: disable=global-statement
|
||||
Fore = colorama.Fore
|
||||
Back = colorama.Back
|
||||
Style = colorama.Style
|
||||
@@ -0,0 +1,963 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import ast as stdlib_ast
|
||||
import functools
|
||||
import sys
|
||||
from contextlib import contextmanager, suppress
|
||||
from copy import copy
|
||||
|
||||
from . import buffering, color, tokenizing
|
||||
from .ast import AST
|
||||
from .collections import OrderedSet as oset
|
||||
from .exceptions import (
|
||||
FailedCut,
|
||||
FailedExpectingEndOfText,
|
||||
FailedKeywordSemantics,
|
||||
FailedLeftRecursion,
|
||||
FailedLookahead,
|
||||
FailedParse,
|
||||
FailedPattern,
|
||||
FailedRef,
|
||||
FailedSemantics,
|
||||
FailedToken,
|
||||
OptionSucceeded,
|
||||
ParseError,
|
||||
)
|
||||
from .infos import (
|
||||
Alert,
|
||||
MemoKey,
|
||||
ParseInfo,
|
||||
ParserConfig,
|
||||
ParseState,
|
||||
RuleInfo,
|
||||
RuleResult,
|
||||
)
|
||||
from .tokenizing import Tokenizer
|
||||
from .util import (
|
||||
info,
|
||||
is_list,
|
||||
left_assoc,
|
||||
prune_dict,
|
||||
right_assoc,
|
||||
safe_name,
|
||||
trim,
|
||||
)
|
||||
from .util.unicode_characters import (
|
||||
C_CUT,
|
||||
C_ENTRY,
|
||||
C_FAILURE,
|
||||
C_RECURSION,
|
||||
C_SUCCESS,
|
||||
)
|
||||
|
||||
__all__ = ['ParseContext', 'tatsumasu', 'leftrec', 'nomemo']
|
||||
|
||||
|
||||
# decorator for rule implementation methods
|
||||
def tatsumasu(*params, **kwparams):
|
||||
def decorator(impl):
|
||||
@functools.wraps(impl)
|
||||
def wrapper(self):
|
||||
name = impl.__name__
|
||||
# remove the single leading and trailing underscore
|
||||
# that the parser generator added
|
||||
name = name[1:-1]
|
||||
is_leftrec = getattr(impl, 'is_leftrec', False)
|
||||
is_memoizable = getattr(impl, 'is_memoizable', True)
|
||||
is_name = getattr(impl, 'is_name', False)
|
||||
ruleinfo = RuleInfo(
|
||||
name,
|
||||
impl,
|
||||
is_leftrec,
|
||||
is_memoizable,
|
||||
is_name,
|
||||
params,
|
||||
kwparams,
|
||||
)
|
||||
return self._call(ruleinfo)
|
||||
|
||||
return wrapper
|
||||
|
||||
return decorator
|
||||
|
||||
|
||||
# This is used to mark left recursive rules
|
||||
def leftrec(impl):
|
||||
impl.is_leftrec = True
|
||||
impl.is_memoizable = False
|
||||
return impl
|
||||
|
||||
|
||||
# Marks rules for which memoization has to be turned off
|
||||
# (has no effect when left recursion is turned off)
|
||||
def nomemo(impl):
|
||||
impl.is_memoizable = False
|
||||
return impl
|
||||
|
||||
|
||||
# Marks rules marked as @name in the grammar
|
||||
def isname(impl):
|
||||
impl.is_name = True
|
||||
return impl
|
||||
|
||||
|
||||
class closure(list):
|
||||
def __hash__(self):
|
||||
return hash(tuple(self))
|
||||
|
||||
|
||||
class ParseContext:
|
||||
def __init__(self, /, config: ParserConfig | None = None, **settings):
|
||||
super().__init__()
|
||||
config = ParserConfig.new(config, **settings)
|
||||
self.config = config
|
||||
self._active_config = self.config
|
||||
|
||||
self._tokenizer: Tokenizer | None = None
|
||||
|
||||
self._semantics = config.semantics
|
||||
|
||||
self._initialize_caches()
|
||||
|
||||
def _initialize_caches(self):
|
||||
self._statestack = [ParseState()]
|
||||
self._rule_stack = []
|
||||
self._cut_stack = [False]
|
||||
|
||||
self._last_node = None
|
||||
self.substate = None
|
||||
self._lookahead = 0
|
||||
self._furthest_exception = None
|
||||
|
||||
self._clear_memoization_caches()
|
||||
|
||||
@property
|
||||
def active_config(self):
|
||||
return self._active_config
|
||||
|
||||
@property
|
||||
def semantics(self):
|
||||
return self._semantics
|
||||
|
||||
@property
|
||||
def encoding(self):
|
||||
return self.active_config.encoding
|
||||
|
||||
@property
|
||||
def parseinfo(self):
|
||||
return self.active_config.parseinfo
|
||||
|
||||
@property
|
||||
def trace(self):
|
||||
return self.active_config.trace
|
||||
|
||||
@property
|
||||
def trace_length(self):
|
||||
return self.active_config.trace_length
|
||||
|
||||
@property
|
||||
def trace_separator(self):
|
||||
return self.active_config.trace_separator
|
||||
|
||||
@property
|
||||
def trace_filename(self):
|
||||
return self.active_config.trace_filename
|
||||
|
||||
@property
|
||||
def comments_re(self):
|
||||
return self.active_config.comments_re
|
||||
|
||||
@property
|
||||
def eol_comments_re(self):
|
||||
return self.active_config.eol_comments_re
|
||||
|
||||
@property
|
||||
def whitespace(self):
|
||||
return self.active_config.whitespace
|
||||
|
||||
@property
|
||||
def ignorecase(self):
|
||||
return self.active_config.ignorecase
|
||||
|
||||
@property
|
||||
def nameguard(self):
|
||||
return self.active_config.nameguard
|
||||
|
||||
@property
|
||||
def memoize_lookaheads(self):
|
||||
return self.active_config.memoize_lookaheads
|
||||
|
||||
@property
|
||||
def left_recursion(self):
|
||||
return self.active_config.left_recursion
|
||||
|
||||
@property
|
||||
def colorize(self):
|
||||
return self.active_config.colorize
|
||||
|
||||
@property
|
||||
def keywords(self):
|
||||
return self._keywords
|
||||
|
||||
@property
|
||||
def namechars(self):
|
||||
return self.active_config.namechars
|
||||
|
||||
def _reset(self, config: ParserConfig):
|
||||
if self.active_config.colorize:
|
||||
color.init()
|
||||
self._initialize_caches()
|
||||
self._keywords = oset(config.keywords)
|
||||
self._semantics = config.semantics
|
||||
if hasattr(self.semantics, 'set_context'):
|
||||
self.semantics.set_context(self)
|
||||
return config
|
||||
|
||||
def _set_furthest_exception(self, e):
|
||||
if (
|
||||
not self._furthest_exception
|
||||
or e.pos > self._furthest_exception.pos
|
||||
):
|
||||
self._furthest_exception = e
|
||||
|
||||
def parse(self, text, /, config: ParserConfig | None = None, **settings):
|
||||
config = self.config.replace_config(config)
|
||||
config = config.replace(**settings)
|
||||
self._active_config = config
|
||||
|
||||
self._reset(config)
|
||||
if isinstance(text, tokenizing.Tokenizer):
|
||||
tokenizer = text
|
||||
elif text is not None:
|
||||
cls = self.tokenizercls
|
||||
tokenizer = cls(text, config=config)
|
||||
else:
|
||||
raise ParseError('No tokenizer or text')
|
||||
self._tokenizer = tokenizer
|
||||
start = self.active_config.effective_rule_name() or 'start'
|
||||
|
||||
try:
|
||||
rule = self._find_rule(start)
|
||||
return rule()
|
||||
except FailedCut as e:
|
||||
self._set_furthest_exception(e.nested)
|
||||
raise self._furthest_exception from e
|
||||
except FailedParse as e:
|
||||
self._set_furthest_exception(e)
|
||||
raise self._furthest_exception from e
|
||||
finally:
|
||||
self._clear_memoization_caches()
|
||||
|
||||
@property
|
||||
def tokenizer(self):
|
||||
return self._tokenizer
|
||||
|
||||
@property
|
||||
def tokenizercls(self) -> type[Tokenizer]:
|
||||
if self.config.tokenizercls is None:
|
||||
return buffering.Buffer
|
||||
else:
|
||||
return self.config.tokenizercls
|
||||
|
||||
@property
|
||||
def last_node(self):
|
||||
return self._last_node
|
||||
|
||||
@last_node.setter
|
||||
def last_node(self, value):
|
||||
self._last_node = value
|
||||
|
||||
@property
|
||||
def _pos(self):
|
||||
return self._tokenizer.pos
|
||||
|
||||
def _clear_memoization_caches(self):
|
||||
self._memos = {}
|
||||
self._results = {}
|
||||
self._recursion_depth = 0
|
||||
|
||||
def _goto(self, pos):
|
||||
self._tokenizer.goto(pos)
|
||||
|
||||
def _next(self):
|
||||
return self._tokenizer.next()
|
||||
|
||||
def _next_token(self, ruleinfo=None):
|
||||
if ruleinfo is None or not ruleinfo.name.lstrip('_')[:1].isupper():
|
||||
self._tokenizer.next_token()
|
||||
|
||||
def _define(self, keys, list_keys=None):
|
||||
# if self.ast and isinstance(self.ast, AST):
|
||||
ast = AST()
|
||||
ast._define(keys, list_keys=list_keys)
|
||||
ast.update(self.ast)
|
||||
self.ast = ast
|
||||
|
||||
@property
|
||||
def state(self):
|
||||
return self._statestack[-1]
|
||||
|
||||
@property
|
||||
def ast(self):
|
||||
return self.state.ast
|
||||
|
||||
@ast.setter
|
||||
def ast(self, value):
|
||||
self.state.ast = value
|
||||
|
||||
def name_last_node(self, name):
|
||||
self.ast[name] = self.last_node
|
||||
|
||||
def add_last_node_to_name(self, name):
|
||||
self.ast._setlist(name, self.last_node)
|
||||
|
||||
@staticmethod
|
||||
def _safe_name(name, ast):
|
||||
while name in ast:
|
||||
name += '_'
|
||||
return name
|
||||
|
||||
def ast_set(self, name, value, as_list=False):
|
||||
ast = self.ast
|
||||
name = self._safe_name(name, ast)
|
||||
|
||||
previous = ast.get(name)
|
||||
if previous is None:
|
||||
new_value = [value] if as_list else value
|
||||
elif is_list(previous):
|
||||
new_value = [*previous, value]
|
||||
else:
|
||||
new_value = [previous, value]
|
||||
|
||||
ast = AST(ast, name=new_value)
|
||||
self.ast = ast
|
||||
|
||||
def ast_append(self, name, value):
|
||||
self.ast_set(name, value, as_list=True)
|
||||
|
||||
def _push_ast(self, copyast=False):
|
||||
ast = copy(self.ast) if copyast else AST()
|
||||
self.state.pos = self._pos
|
||||
self._statestack.append(ParseState(ast=ast, pos=self._pos))
|
||||
|
||||
def _pop_ast(self):
|
||||
self._statestack.pop()
|
||||
self.tokenizer.goto(self.state.pos)
|
||||
|
||||
def _merge_ast(self):
|
||||
pos = self._pos
|
||||
ast = self.ast
|
||||
cst = self.cst
|
||||
self._statestack.pop()
|
||||
self.ast = ast
|
||||
self._extend_cst(cst)
|
||||
self.tokenizer.goto(pos)
|
||||
|
||||
@property
|
||||
def cst(self):
|
||||
return self.state.cst
|
||||
|
||||
@cst.setter
|
||||
def cst(self, value):
|
||||
self.state.cst = value
|
||||
|
||||
def _push_cst(self):
|
||||
self._statestack.append(ParseState(ast=self.ast))
|
||||
|
||||
def _pop_cst(self):
|
||||
ast = self.ast
|
||||
self._statestack.pop()
|
||||
self.ast = ast
|
||||
|
||||
def _merge_cst(self, extend=True):
|
||||
cst = self.cst
|
||||
self._pop_cst()
|
||||
if extend:
|
||||
self._extend_cst(cst)
|
||||
else:
|
||||
self._append_cst(cst)
|
||||
return cst
|
||||
|
||||
def _append_cst(self, node):
|
||||
self.last_node = node
|
||||
previous = self.cst
|
||||
if previous is None:
|
||||
self.cst = self._copy_node(node)
|
||||
elif is_list(previous):
|
||||
previous.append(node)
|
||||
else:
|
||||
self.cst = [previous, node]
|
||||
return node
|
||||
|
||||
def _extend_cst(self, node):
|
||||
self.last_node = node
|
||||
if node is None:
|
||||
return None
|
||||
previous = self.cst
|
||||
if previous is None:
|
||||
self.cst = self._copy_node(node)
|
||||
elif is_list(node):
|
||||
if is_list(previous):
|
||||
previous.extend(node)
|
||||
else:
|
||||
self.cst = [previous, *node]
|
||||
elif is_list(previous):
|
||||
previous.append(node)
|
||||
else:
|
||||
self.cst = [previous, node]
|
||||
return node
|
||||
|
||||
def _copy_node(self, node):
|
||||
if node is None:
|
||||
return None
|
||||
elif is_list(node):
|
||||
return node[:]
|
||||
else:
|
||||
return node
|
||||
|
||||
def _is_cut_set(self):
|
||||
return self._cut_stack[-1]
|
||||
|
||||
def _cut(self):
|
||||
self._trace_cut()
|
||||
self._cut_stack[-1] = True
|
||||
|
||||
# Kota Mizushima et al say that we can throw away
|
||||
# memos for previous positions in the tokenizer under
|
||||
# certain circumstances, without affecting the linearity
|
||||
# of PEG parsing.
|
||||
# https://kmizu.github.io/papers/paste513-mizushima.pdf
|
||||
#
|
||||
# We adopt the heuristic of always dropping the cache for
|
||||
# positions less than the current cut position. It remains to
|
||||
# be proven if doing it this way affects linearity. Empirically,
|
||||
# it hasn't.
|
||||
|
||||
def prune(cache, cutpos):
|
||||
prune_dict(cache, lambda k, _: k[0] < cutpos)
|
||||
|
||||
prune(self._memos, self._pos)
|
||||
|
||||
def _memoization(self):
|
||||
return self.memoize_lookaheads or self._lookahead == 0
|
||||
|
||||
def _rulestack(self):
|
||||
rulestack = [r.name for r in reversed(self._rule_stack)]
|
||||
stack = self.trace_separator.join(rulestack)
|
||||
if max(len(s) for s in stack.splitlines()) > self.trace_length:
|
||||
stack = stack[: self.trace_length]
|
||||
stack = stack.rsplit(self.trace_separator, 1)[0]
|
||||
stack += self.trace_separator
|
||||
return stack
|
||||
|
||||
def _find_rule(self, name):
|
||||
self._error(name, exclass=FailedRef)
|
||||
return lambda: None # makes static checkers happy
|
||||
|
||||
def _find_semantic_action(self, name):
|
||||
if self.semantics is None:
|
||||
return None, None
|
||||
|
||||
postproc = getattr(self.semantics, '_postproc', None)
|
||||
|
||||
action = getattr(self.semantics, safe_name(name), None)
|
||||
if not callable(action):
|
||||
action = getattr(self.semantics, '_default', None)
|
||||
|
||||
if not callable(action):
|
||||
action = None
|
||||
if not callable(postproc):
|
||||
postproc = None
|
||||
|
||||
return action, postproc
|
||||
|
||||
def _trace(self, msg, *params, **kwargs):
|
||||
if self.trace:
|
||||
msg %= params
|
||||
info(str(msg), file=sys.stderr)
|
||||
|
||||
def _trace_event(self, event):
|
||||
if self.trace:
|
||||
fname = ''
|
||||
if self.trace_filename:
|
||||
fname = self._tokenizer.line_info().filename + '\n'
|
||||
|
||||
lookahead = self._tokenizer.lookahead().rstrip()
|
||||
if lookahead:
|
||||
lookahead = '\n' + lookahead
|
||||
self._trace(
|
||||
'%s %s%s%s',
|
||||
event + self._rulestack(),
|
||||
self._tokenizer.lookahead_pos(),
|
||||
color.Style.DIM + fname,
|
||||
color.Style.NORMAL + lookahead + color.Style.RESET_ALL,
|
||||
end='',
|
||||
)
|
||||
|
||||
def _trace_entry(self):
|
||||
self._trace_event(color.Fore.YELLOW + color.Style.BRIGHT + C_ENTRY)
|
||||
|
||||
def _trace_success(self):
|
||||
self._trace_event(color.Fore.GREEN + color.Style.BRIGHT + C_SUCCESS)
|
||||
|
||||
def _trace_failure(self, ex=None):
|
||||
if isinstance(ex, FailedLeftRecursion):
|
||||
self._trace_recursion()
|
||||
else:
|
||||
self._trace_event(color.Fore.RED + color.Style.BRIGHT + C_FAILURE)
|
||||
|
||||
def _trace_recursion(self):
|
||||
self._trace_event(color.Fore.RED + color.Style.BRIGHT + C_RECURSION)
|
||||
|
||||
def _trace_cut(self):
|
||||
self._trace_event(color.Fore.MAGENTA + color.Style.BRIGHT + C_CUT)
|
||||
|
||||
def _trace_match(self, token, name=None, failed=False):
|
||||
if self.trace:
|
||||
fname = ''
|
||||
if self.trace_filename:
|
||||
fname = self._tokenizer.line_info().filename + '\n'
|
||||
name = f'/{name}/' if name else ''
|
||||
|
||||
if not failed:
|
||||
fgcolor = color.Fore.GREEN + C_SUCCESS
|
||||
else:
|
||||
fgcolor = color.Fore.RED + C_FAILURE
|
||||
|
||||
lookahead = self._tokenizer.lookahead().rstrip()
|
||||
if lookahead:
|
||||
lookahead = '\n' + lookahead
|
||||
|
||||
self._trace(
|
||||
color.Style.BRIGHT + fgcolor + "'%s' %s%s%s",
|
||||
token,
|
||||
name,
|
||||
color.Style.DIM + fname,
|
||||
color.Style.NORMAL + lookahead + color.Style.RESET_ALL,
|
||||
end='',
|
||||
)
|
||||
|
||||
def _make_exception(self, item, exclass=FailedParse):
|
||||
rulestack = (r.name for r in self._rule_stack[::-1])
|
||||
return exclass(self.tokenizer, rulestack, item)
|
||||
|
||||
def _error(self, item, exclass=FailedParse):
|
||||
raise self._make_exception(item, exclass=exclass)
|
||||
|
||||
def _fail(self):
|
||||
self._error('fail')
|
||||
|
||||
def _get_parseinfo(self, name, pos):
|
||||
endpos = self._pos
|
||||
return ParseInfo(
|
||||
tokenizer=self.tokenizer,
|
||||
rule=name,
|
||||
pos=pos,
|
||||
endpos=endpos,
|
||||
line=self.tokenizer.posline(pos),
|
||||
endline=self.tokenizer.posline(endpos),
|
||||
alerts=self.state.alerts,
|
||||
)
|
||||
|
||||
@property
|
||||
def rule(self):
|
||||
return self._rule_stack[-1]
|
||||
|
||||
@property
|
||||
def memokey(self):
|
||||
return MemoKey(self._pos, self.rule, self.substate)
|
||||
|
||||
def _memoize(self, key, memo):
|
||||
if self._memoization() and key.rule.is_memoizable:
|
||||
self._memos[key] = memo
|
||||
return memo
|
||||
|
||||
def _forget(self, key):
|
||||
self._memos.pop(key, None)
|
||||
|
||||
def _memo_for(self, key):
|
||||
return self._memos.get(key)
|
||||
|
||||
# if isinstance(memo, FailedLeftRecursion):
|
||||
# memo = self._results.get(key, memo)
|
||||
|
||||
def _mkresult(self, node):
|
||||
return RuleResult(node, self._pos, self.substate)
|
||||
|
||||
def _save_result(self, key, result):
|
||||
if is_list(result.node):
|
||||
result = RuleResult(
|
||||
closure(result.node), result.newpos, result.newstate,
|
||||
)
|
||||
self._results[key] = result
|
||||
|
||||
def _is_recursive(self, ruleinfo):
|
||||
return ruleinfo.is_leftrec
|
||||
|
||||
def _set_left_recursion_guard(self, key):
|
||||
ex = self._make_exception(key.rule.name, exclass=FailedLeftRecursion)
|
||||
self._memoize(key, ex)
|
||||
|
||||
def _call(self, ruleinfo):
|
||||
self._rule_stack += [ruleinfo]
|
||||
pos = self._pos
|
||||
try:
|
||||
self._trace_entry()
|
||||
|
||||
self._last_node = None
|
||||
|
||||
result = self._recursive_call(ruleinfo)
|
||||
node, newpos, newstate = result
|
||||
|
||||
self._goto(newpos)
|
||||
self.substate = newstate
|
||||
self._append_cst(node)
|
||||
|
||||
self._trace_success()
|
||||
|
||||
return node
|
||||
except FailedPattern:
|
||||
self._error(f'Expecting <{ruleinfo.name}>')
|
||||
except FailedParse as e:
|
||||
self._goto(pos)
|
||||
self._set_furthest_exception(e)
|
||||
self._trace_failure(e)
|
||||
raise
|
||||
finally:
|
||||
self._rule_stack.pop()
|
||||
|
||||
def _clear_recursion_errors(self):
|
||||
def filter(key, value):
|
||||
return isinstance(value, FailedLeftRecursion)
|
||||
|
||||
prune_dict(self._memos, filter)
|
||||
|
||||
def _recursive_call(self, ruleinfo):
|
||||
self._next_token(ruleinfo)
|
||||
key = self.memokey
|
||||
|
||||
if not ruleinfo.is_leftrec:
|
||||
return self._invoke_rule(ruleinfo, key)
|
||||
elif not self.left_recursion:
|
||||
self._error('Left recursion detected', exclass=FailedLeftRecursion)
|
||||
|
||||
self._recursion_depth += 1
|
||||
if key in self._results:
|
||||
result = self._results[key]
|
||||
else:
|
||||
result = self._make_exception(
|
||||
ruleinfo.name, exclass=FailedLeftRecursion,
|
||||
)
|
||||
self._results[key] = result
|
||||
|
||||
initial = self._pos
|
||||
lastpos = initial - 1
|
||||
while True:
|
||||
try:
|
||||
self._clear_recursion_errors()
|
||||
new_result = self._invoke_rule(ruleinfo, key)
|
||||
self._goto(initial)
|
||||
except FailedParse:
|
||||
break
|
||||
|
||||
if new_result.newpos > lastpos:
|
||||
self._save_result(key, new_result)
|
||||
lastpos = new_result.newpos
|
||||
result = new_result
|
||||
else:
|
||||
break
|
||||
self._recursion_depth -= 1
|
||||
|
||||
if isinstance(result, Exception):
|
||||
raise result # pylint: disable=raising-non-exception
|
||||
|
||||
return result
|
||||
|
||||
def _invoke_rule(self, ruleinfo, key):
|
||||
memo = self._memo_for(key)
|
||||
if isinstance(memo, Exception):
|
||||
raise memo
|
||||
if memo:
|
||||
return memo
|
||||
self._set_left_recursion_guard(key)
|
||||
|
||||
self._push_ast()
|
||||
try:
|
||||
try:
|
||||
self._next_token(ruleinfo)
|
||||
ruleinfo.impl(self)
|
||||
node = self._get_node(key.pos, ruleinfo)
|
||||
node = self._invoke_semantic_rule(ruleinfo, node)
|
||||
result = self._mkresult(node)
|
||||
self._memoize(key, result)
|
||||
return result
|
||||
except FailedSemantics as e:
|
||||
self._error(str(e))
|
||||
except FailedParse as e:
|
||||
self._memoize(key, e)
|
||||
raise
|
||||
finally:
|
||||
self._pop_ast()
|
||||
|
||||
def _get_node(self, pos, ruleinfo):
|
||||
node = self.ast
|
||||
if not node:
|
||||
node = tuple(self.cst) if is_list(self.cst) else self.cst
|
||||
elif '@' in node:
|
||||
node = node['@'] # override the AST
|
||||
elif self.parseinfo:
|
||||
node.set_parseinfo(self._get_parseinfo(ruleinfo.name, pos))
|
||||
return node
|
||||
|
||||
def _invoke_semantic_rule(self, rule, node):
|
||||
semantic_rule, postproc = self._find_semantic_action(rule.name)
|
||||
if semantic_rule:
|
||||
node = semantic_rule(
|
||||
node, *(rule.params or ()), **(rule.kwparams or {}),
|
||||
)
|
||||
if callable(postproc):
|
||||
postproc(self, node) # pylint: disable=not-callable
|
||||
if rule.is_name:
|
||||
self._check_name(node)
|
||||
return node
|
||||
|
||||
def _token(self, token):
|
||||
self._next_token()
|
||||
if self.tokenizer.match(token) is None:
|
||||
self._trace_match(token, failed=True)
|
||||
self._error(token, exclass=FailedToken)
|
||||
self._trace_match(token)
|
||||
self._append_cst(token)
|
||||
return token
|
||||
|
||||
def _constant(self, literal):
|
||||
self._next_token()
|
||||
self._trace_match(literal)
|
||||
if isinstance(literal, str):
|
||||
try:
|
||||
literal = stdlib_ast.literal_eval(literal.strip())
|
||||
except (ValueError, SyntaxError):
|
||||
if '\n' in literal:
|
||||
literal = trim(literal)
|
||||
literal = eval( # noqa: S307
|
||||
f'{"f" + repr(literal)}', {}, self.ast,
|
||||
)
|
||||
self._append_cst(literal)
|
||||
return literal
|
||||
|
||||
def _alert(self, message, level):
|
||||
self._next_token()
|
||||
self._trace_match(f'{"^" * level}`{message}`', failed=True)
|
||||
self.state.alerts.append(Alert(message=message, level=level))
|
||||
|
||||
def _pattern(self, pattern):
|
||||
token = self.tokenizer.matchre(pattern)
|
||||
if token is None:
|
||||
self._trace_match('', pattern, failed=True)
|
||||
self._error(pattern, exclass=FailedPattern)
|
||||
self._trace_match(token, pattern)
|
||||
self._append_cst(token)
|
||||
return token
|
||||
|
||||
def _eof(self):
|
||||
return self.tokenizer.atend()
|
||||
|
||||
def _eol(self):
|
||||
return self.tokenizer.ateol()
|
||||
|
||||
def _check_eof(self):
|
||||
self._next_token()
|
||||
if not self.tokenizer.atend():
|
||||
self._error(
|
||||
'Expecting end of text', exclass=FailedExpectingEndOfText,
|
||||
)
|
||||
|
||||
@contextmanager
|
||||
def _try(self):
|
||||
s = self.substate
|
||||
self._push_ast(copyast=True)
|
||||
self.last_node = None
|
||||
try:
|
||||
yield
|
||||
self._merge_ast()
|
||||
except FailedParse:
|
||||
self._pop_ast()
|
||||
self.substate = s
|
||||
raise
|
||||
|
||||
@contextmanager
|
||||
def _option(self):
|
||||
self.last_node = None
|
||||
self._cut_stack += [False]
|
||||
try:
|
||||
with self._try():
|
||||
yield
|
||||
raise OptionSucceeded()
|
||||
except FailedCut:
|
||||
raise
|
||||
except FailedParse as e:
|
||||
if self._is_cut_set():
|
||||
raise FailedCut(e) from e
|
||||
finally:
|
||||
self._cut_stack.pop()
|
||||
|
||||
@contextmanager
|
||||
def _choice(self):
|
||||
self.last_node = None
|
||||
with suppress(OptionSucceeded):
|
||||
yield
|
||||
|
||||
@contextmanager
|
||||
def _optional(self):
|
||||
self.last_node = None
|
||||
with self._choice(), self._option():
|
||||
yield
|
||||
|
||||
@contextmanager
|
||||
def _group(self):
|
||||
self._push_cst()
|
||||
try:
|
||||
yield
|
||||
self._merge_cst(extend=True)
|
||||
except Exception:
|
||||
self._pop_cst()
|
||||
raise
|
||||
|
||||
@contextmanager
|
||||
def _if(self):
|
||||
s = self.substate
|
||||
self._push_ast()
|
||||
self._lookahead += 1
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self._pop_ast() # simply discard
|
||||
self._lookahead -= 1
|
||||
self.substate = s
|
||||
|
||||
@contextmanager
|
||||
def _ifnot(self):
|
||||
try:
|
||||
with self._if():
|
||||
yield
|
||||
except FailedParse:
|
||||
pass
|
||||
else:
|
||||
self._error('', exclass=FailedLookahead)
|
||||
|
||||
def _isolate(self, block, drop=False):
|
||||
self._push_cst()
|
||||
try:
|
||||
block()
|
||||
cst = self.cst
|
||||
finally:
|
||||
self._pop_cst()
|
||||
|
||||
if is_list(cst):
|
||||
cst = closure(cst)
|
||||
if not drop:
|
||||
self._append_cst(cst)
|
||||
return cst
|
||||
|
||||
def _repeat(self, block, prefix=None, dropprefix=False):
|
||||
while True:
|
||||
with self._choice():
|
||||
with self._option():
|
||||
p = self._pos
|
||||
|
||||
if prefix:
|
||||
self._isolate(prefix, drop=dropprefix)
|
||||
self._cut()
|
||||
|
||||
self._isolate(block)
|
||||
|
||||
if self._pos == p:
|
||||
self._error('empty closure')
|
||||
return
|
||||
|
||||
def _closure(self, block, sep=None, omitsep=False):
|
||||
self._push_cst()
|
||||
try:
|
||||
self.cst = []
|
||||
with self._optional():
|
||||
block()
|
||||
self.cst = [self.cst]
|
||||
self._repeat(block, prefix=sep, dropprefix=omitsep)
|
||||
self.cst = closure(self.cst)
|
||||
return self._merge_cst()
|
||||
except Exception:
|
||||
self._pop_cst()
|
||||
raise
|
||||
|
||||
def _positive_closure(self, block, sep=None, omitsep=False):
|
||||
self._push_cst()
|
||||
try:
|
||||
block()
|
||||
self.cst = [self.cst]
|
||||
self._repeat(block, prefix=sep, dropprefix=omitsep)
|
||||
self.cst = closure(self.cst)
|
||||
return self._merge_cst()
|
||||
except Exception:
|
||||
self._pop_cst()
|
||||
raise
|
||||
|
||||
def _empty_closure(self):
|
||||
cst = closure([])
|
||||
self._append_cst(cst)
|
||||
return cst
|
||||
|
||||
def _gather(self, block, sep):
|
||||
return self._closure(block, sep=sep, omitsep=True)
|
||||
|
||||
def _positive_gather(self, block, sep):
|
||||
return self._positive_closure(block, sep=sep, omitsep=True)
|
||||
|
||||
def _join(self, block, sep):
|
||||
return self._closure(block, sep=sep)
|
||||
|
||||
def _positive_join(self, block, sep):
|
||||
return self._positive_closure(block, sep=sep)
|
||||
|
||||
def _left_join(self, block, sep):
|
||||
self.cst = left_assoc(self._positive_join(block, sep))
|
||||
self.last_node = self.cst
|
||||
return self.cst
|
||||
|
||||
def _right_join(self, block, sep):
|
||||
self.cst = right_assoc(self._positive_join(block, sep))
|
||||
self.last_node = self.cst
|
||||
return self.cst
|
||||
|
||||
def _check_name(self, name):
|
||||
name = str(name)
|
||||
if self.ignorecase or self.tokenizer.ignorecase:
|
||||
name = name.upper()
|
||||
if name in self.keywords:
|
||||
raise FailedKeywordSemantics(f'"{name}" is a reserved word')
|
||||
|
||||
def _void(self):
|
||||
self.last_node = None
|
||||
|
||||
def _any(self):
|
||||
c = self._next()
|
||||
if c is None:
|
||||
self._trace_match(c, failed=True)
|
||||
self._error(c, exclass=FailedToken)
|
||||
self._trace_match(c)
|
||||
self._append_cst(c)
|
||||
return c
|
||||
|
||||
def _skip_to(self, block):
|
||||
while not self._eof():
|
||||
try:
|
||||
with self._if():
|
||||
block()
|
||||
except FailedParse:
|
||||
pass
|
||||
else:
|
||||
break
|
||||
pos = self._pos
|
||||
self._next_token()
|
||||
if self._pos == pos:
|
||||
self._next()
|
||||
block()
|
||||
@@ -0,0 +1,282 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import itertools
|
||||
|
||||
import pygraphviz as pgv # pylint: disable=E0401
|
||||
|
||||
from .walkers import NodeWalker
|
||||
|
||||
__all__ = ['draw']
|
||||
|
||||
|
||||
def draw(filename, grammar):
|
||||
traverser = GraphvizWalker()
|
||||
traverser.walk(grammar)
|
||||
traverser.draw(filename)
|
||||
|
||||
|
||||
class GraphvizWalker(NodeWalker):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.top_graph = pgv.AGraph(
|
||||
directed=True, rankdir='LR', packMode='clust', splines='true',
|
||||
)
|
||||
self.stack = [self.top_graph]
|
||||
self.node_count = 0
|
||||
|
||||
@property
|
||||
def graph(self):
|
||||
return self.stack[-1]
|
||||
|
||||
def draw(self, filename):
|
||||
self.graph.layout(prog='dot')
|
||||
# WARNING: neato generated graphics hang my GPU
|
||||
# self.graph.layout(prog='neato')
|
||||
self.graph.draw(filename)
|
||||
|
||||
def push_graph(self, name=None, **attr):
|
||||
if name is None:
|
||||
self.node_count += 1
|
||||
name = 'g%d' % self.node_count
|
||||
self.stack.append(self.graph.add_subgraph(name, **attr))
|
||||
return self.graph
|
||||
|
||||
def pop_graph(self):
|
||||
self.stack.pop()
|
||||
|
||||
def node(self, name, id=None, **attr):
|
||||
if id is None:
|
||||
self.node_count += 1
|
||||
id = 'n%d' % self.node_count
|
||||
else:
|
||||
try:
|
||||
return self.graph.get_node(id)
|
||||
except KeyError:
|
||||
pass
|
||||
self.graph.add_node(id, **attr)
|
||||
n = self.graph.get_node(id)
|
||||
n.attr['label'] = name
|
||||
# n.attr['shape'] = 'circle'
|
||||
return n
|
||||
|
||||
def tnode(self, name, **attr):
|
||||
return self.node(name, **attr)
|
||||
|
||||
def dot(self):
|
||||
n = self.node('')
|
||||
n.attr['shape'] = 'point'
|
||||
n.attr['size'] = 0.0000000001
|
||||
n.attr['label'] = ''
|
||||
return n
|
||||
|
||||
def start_node(self):
|
||||
return self.dot()
|
||||
|
||||
def ref_node(self, name):
|
||||
n = self.node(name)
|
||||
n.attr['shape'] = 'box'
|
||||
return n
|
||||
|
||||
def rule_node(self, name, **attr):
|
||||
n = self.node(name, **attr)
|
||||
n.attr['shape'] = 'parallelogram'
|
||||
return n
|
||||
|
||||
def end_node(self):
|
||||
n = self.node('')
|
||||
n.attr['shape'] = 'point'
|
||||
n.attr['width'] = 0.1
|
||||
return n
|
||||
|
||||
def edge(self, s, e, **attr):
|
||||
self.graph.add_edge(s, e, **attr)
|
||||
edge = self.graph.get_edge(s, e)
|
||||
# edge.attr['arrowhead'] = 'normal'
|
||||
edge.attr['arrowhead'] = 'none'
|
||||
return edge
|
||||
|
||||
def redge(self, s, e):
|
||||
edge = self.edge(s, e)
|
||||
edge.attr['dir'] = 'back'
|
||||
return edge
|
||||
|
||||
def zedge(self, s, e):
|
||||
return self.edge(s, e, len=0.000001)
|
||||
|
||||
def nedge(self, s, e):
|
||||
return self.edge(s, e, style='invisible', dir='none')
|
||||
|
||||
def path(self, p):
|
||||
self.graph.add_path(p)
|
||||
|
||||
def subgraph(self, name, bunch):
|
||||
self.top_graph.add_subgraph(name)
|
||||
|
||||
def concat(self, *args):
|
||||
return list(itertools.chain(*args))
|
||||
|
||||
def _walk_decorator(self, d):
|
||||
return self.walk(d.exp)
|
||||
|
||||
def walk_default(self, node):
|
||||
raise NotImplementedError('No walking for ', type(node).__name__)
|
||||
|
||||
def walk__decorator(self, d):
|
||||
return self.walk(d.exp)
|
||||
|
||||
def walk__grammar(self, g):
|
||||
self.push_graph(g.name + '0')
|
||||
try:
|
||||
vrules = [self.walk(r) for r in reversed(g.rules)]
|
||||
finally:
|
||||
self.pop_graph()
|
||||
self.push_graph(g.name + '1')
|
||||
try:
|
||||
pass
|
||||
# link all rule starting nodes with invisible edges
|
||||
# starts = [self.node(r.name, id=r.name) for r in g.rules]
|
||||
# for n1, n2 in zip(starts, starts[1:]):
|
||||
# self.nedge(n1, n2)
|
||||
finally:
|
||||
self.pop_graph()
|
||||
s, t = vrules[0][0], vrules[-1][1]
|
||||
return (s, t)
|
||||
|
||||
def walk__rule(self, r):
|
||||
self.push_graph(r.name)
|
||||
try:
|
||||
i, e = self.walk(r.exp)
|
||||
s = self.rule_node(r.name, id=r.name)
|
||||
self.edge(s, i)
|
||||
t = self.end_node()
|
||||
self.edge(e, t)
|
||||
return (s, t)
|
||||
finally:
|
||||
self.pop_graph()
|
||||
|
||||
def walk__based_rule(self, r):
|
||||
return self.walk__rule(r)
|
||||
|
||||
def walk__rule_ref(self, rr):
|
||||
n = self.ref_node(rr.name)
|
||||
return (n, n)
|
||||
|
||||
def walk__special(self, s):
|
||||
n = self.node(s.special)
|
||||
return (n, n)
|
||||
|
||||
def walk__override(self, o):
|
||||
return self._walk_decorator(o)
|
||||
|
||||
def walk__named(self, n):
|
||||
return self._walk_decorator(n)
|
||||
|
||||
def walk__named_list(self, n):
|
||||
return self._walk_decorator(n)
|
||||
|
||||
def walk__cut(self, c):
|
||||
# c = self.node('>>')
|
||||
# return (c, c)
|
||||
return None
|
||||
|
||||
def walk__optional(self, o):
|
||||
i, e = self._walk_decorator(o)
|
||||
ni = self.dot()
|
||||
ne = self.dot()
|
||||
self.zedge(ni, i)
|
||||
self.edge(ni, ne)
|
||||
self.zedge(e, ne)
|
||||
return (ni, ne)
|
||||
|
||||
def walk__closure(self, r):
|
||||
self.push_graph(rankdir='TB')
|
||||
try:
|
||||
i, e = self._walk_decorator(r)
|
||||
ni = self.dot()
|
||||
self.edge(ni, i)
|
||||
self.edge(e, ni)
|
||||
return (ni, ni)
|
||||
finally:
|
||||
self.pop_graph()
|
||||
|
||||
def walk__positive_closure(self, r):
|
||||
i, e = self._walk_decorator(r)
|
||||
if i == e:
|
||||
self.redge(e, i)
|
||||
else:
|
||||
self.edge(e, i)
|
||||
return (i, e)
|
||||
|
||||
def walk__join(self, r):
|
||||
i, e = self._walk_decorator(r)
|
||||
n = self.tnode(r.sep)
|
||||
self.edge(i, n)
|
||||
self.edge(n, e)
|
||||
return (i, e)
|
||||
|
||||
def walk__group(self, g):
|
||||
return self._walk_decorator(g)
|
||||
|
||||
def walk__choice(self, c):
|
||||
vopt = [self.walk(o) for o in c.options]
|
||||
vopt = [o for o in vopt if o is not None]
|
||||
ni = self.dot()
|
||||
ne = self.dot()
|
||||
for i, e in vopt:
|
||||
self.edge(ni, i)
|
||||
self.edge(e, ne)
|
||||
return (ni, ne)
|
||||
|
||||
def walk__sequence(self, s):
|
||||
vseq = [self.walk(x) for x in s.sequence]
|
||||
vseq = [x for x in vseq if x is not None]
|
||||
i, _ = vseq[0]
|
||||
_, e = vseq[-1]
|
||||
if i != e:
|
||||
bunch = zip(
|
||||
[a for _x, a in vseq[:-1]],
|
||||
[b for b, _y in vseq[1:]],
|
||||
strict=False,
|
||||
)
|
||||
for n, n1 in bunch:
|
||||
self.edge(n, n1)
|
||||
return (i, e)
|
||||
|
||||
def walk__lookahead(self, la):
|
||||
_, e = self._walk_decorator(la)
|
||||
n = self.node('&')
|
||||
self.edge(n, e)
|
||||
return (n, e)
|
||||
|
||||
def walk__negative_lookahead(self, la):
|
||||
_, e = self._walk_decorator(la)
|
||||
n = self.node('!')
|
||||
self.edge(n, e)
|
||||
return (n, e)
|
||||
|
||||
def walk__rule_include(self, la):
|
||||
_, e = self._walk_decorator(la)
|
||||
n = self.node('>')
|
||||
self.edge(n, e)
|
||||
return (n, e)
|
||||
|
||||
def walk__pattern(self, p):
|
||||
n = self.tnode(p.pattern)
|
||||
return (n, n)
|
||||
|
||||
def walk__token(self, t):
|
||||
n = self.tnode(t.token)
|
||||
return (n, n)
|
||||
|
||||
def walk__void(self, v):
|
||||
n = self.dot()
|
||||
return (n, n)
|
||||
|
||||
def walk__constant(self, t):
|
||||
n = self.tnode(f'`{t.ast}`')
|
||||
return (n, n)
|
||||
|
||||
def walk__eof(self, v):
|
||||
# n = self.node('$')
|
||||
# return (n, n)
|
||||
return None
|
||||
@@ -0,0 +1,152 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
|
||||
class ParseException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
# alias
|
||||
TatSuException = ParseException
|
||||
|
||||
|
||||
class OptionSucceeded(ParseException):
|
||||
pass
|
||||
|
||||
|
||||
class GrammarError(ParseException):
|
||||
pass
|
||||
|
||||
|
||||
class SemanticError(ParseException):
|
||||
pass
|
||||
|
||||
|
||||
class CodegenError(ParseException):
|
||||
pass
|
||||
|
||||
|
||||
class MissingSemanticFor(SemanticError):
|
||||
pass
|
||||
|
||||
|
||||
class ParseError(ParseException):
|
||||
pass
|
||||
|
||||
|
||||
class FailedSemantics(ParseError):
|
||||
pass
|
||||
|
||||
|
||||
class FailedKeywordSemantics(FailedSemantics):
|
||||
pass
|
||||
|
||||
|
||||
class NoParseInfo(ParseException):
|
||||
pass
|
||||
|
||||
|
||||
class FailedParse(ParseError):
|
||||
def __init__(self, tokenizer, stack, item):
|
||||
stack = list(stack) # NOTE: can't pass through multiprocessing if generator
|
||||
# note: pass all arguments to super() to avoid pickling problems
|
||||
# https://stackoverflow.com/questions/27993567/
|
||||
super().__init__(tokenizer, stack, item)
|
||||
|
||||
self.tokenizer = tokenizer
|
||||
self.stack = stack
|
||||
self.pos = tokenizer.pos
|
||||
self.item = item
|
||||
|
||||
@property
|
||||
def message(self):
|
||||
return self.item
|
||||
|
||||
def __str__(self):
|
||||
info = self.tokenizer.line_info(self.pos)
|
||||
template = '{}({}:{}) {} :\n{}\n{}^\n{}'
|
||||
text = info.text.rstrip()
|
||||
leading = re.sub(r'[^\t]', ' ', text)[: info.col]
|
||||
|
||||
text = text.expandtabs()
|
||||
leading = leading.expandtabs()
|
||||
return template.format(
|
||||
info.filename,
|
||||
info.line + 1,
|
||||
info.col + 1,
|
||||
self.message.rstrip(),
|
||||
text,
|
||||
leading,
|
||||
'\n'.join(self.stack),
|
||||
)
|
||||
|
||||
|
||||
class FailedToken(FailedParse):
|
||||
def __init__(self, tokenizer, stack, token):
|
||||
super().__init__(tokenizer, stack, token)
|
||||
self.token = token
|
||||
|
||||
@property
|
||||
def message(self):
|
||||
return 'expecting {}'.format(repr(self.token).lstrip('u'))
|
||||
|
||||
|
||||
class FailedPattern(FailedParse):
|
||||
def __init__(self, tokenizer, stack, pattern):
|
||||
super().__init__(tokenizer, stack, pattern)
|
||||
self.pattern = pattern
|
||||
|
||||
@property
|
||||
def message(self):
|
||||
return f'expecting /{self.pattern}/'
|
||||
|
||||
|
||||
class FailedRef(FailedParse):
|
||||
def __init__(self, tokenizer, stack, name):
|
||||
super().__init__(tokenizer, stack, name)
|
||||
self.name = name
|
||||
|
||||
@property
|
||||
def message(self):
|
||||
return f"could not resolve reference to rule '{self.name}'"
|
||||
|
||||
|
||||
class FailedCut(FailedParse):
|
||||
def __init__(self, nested):
|
||||
super().__init__(nested.tokenizer, nested.stack, nested.item)
|
||||
self.pos = nested.pos
|
||||
self.nested = nested
|
||||
|
||||
@property
|
||||
def message(self):
|
||||
return self.nested.message
|
||||
|
||||
def __reduce__(self):
|
||||
return type(self), (self.nested,)
|
||||
|
||||
|
||||
class FailedChoice(FailedParse):
|
||||
@property
|
||||
def message(self):
|
||||
return 'no viable option'
|
||||
|
||||
|
||||
class FailedLookahead(FailedParse):
|
||||
@property
|
||||
def message(self):
|
||||
return 'failed lookahead'
|
||||
|
||||
|
||||
class FailedLeftRecursion(FailedParse):
|
||||
@property
|
||||
def message(self):
|
||||
return 'infinite left recursion'
|
||||
|
||||
|
||||
class FailedExpectingEndOfText(FailedParse):
|
||||
pass
|
||||
|
||||
|
||||
class FailedKeyword(FailedParse):
|
||||
pass
|
||||
@@ -0,0 +1,58 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import pkgutil
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from tatsu import compile
|
||||
|
||||
from .semantics import ANTLRSemantics
|
||||
|
||||
|
||||
def antlr_grammar():
|
||||
return str(pkgutil.get_data(__name__, 'antlr.ebnf'), 'utf-8')
|
||||
|
||||
|
||||
def translate(
|
||||
text=None, filename=None, name=None, encoding='utf-8', trace=False,
|
||||
):
|
||||
if text is None and filename is None:
|
||||
raise ValueError('either `text` or `filename` must be provided')
|
||||
if filename:
|
||||
filename = Path(filename)
|
||||
|
||||
if text is None:
|
||||
name = name or filename.stem
|
||||
with filename.open(encoding=encoding) as f:
|
||||
text = f.read()
|
||||
|
||||
name = name or 'Unknown'
|
||||
|
||||
semantics = ANTLRSemantics(name)
|
||||
grammar = compile(antlr_grammar())
|
||||
model = grammar.parse(
|
||||
text,
|
||||
name=name,
|
||||
filename=filename,
|
||||
semantics=semantics,
|
||||
trace=trace,
|
||||
colorize=True,
|
||||
)
|
||||
print(model)
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 2:
|
||||
thisprog = Path(sys.argv[0]).name
|
||||
print(thisprog)
|
||||
print('Usage:')
|
||||
print('\t', thisprog, 'FILENAME.g [--trace]')
|
||||
sys.exit(1)
|
||||
translate(
|
||||
filename=sys.argv[1], trace='--trace' in sys.argv or '-t' in sys.argv,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from . import main
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,197 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from itertools import chain
|
||||
|
||||
from tatsu import grammars as model
|
||||
from tatsu.ast import AST
|
||||
|
||||
|
||||
def camel2py(name):
|
||||
return re.sub(
|
||||
r'([a-z0-9])([A-Z])',
|
||||
lambda m: m.group(1) + '_' + m.group(2).lower(),
|
||||
name,
|
||||
)
|
||||
|
||||
|
||||
class ANTLRSemantics:
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
self.tokens = {}
|
||||
self.token_rules = {}
|
||||
self.synthetic_rules = []
|
||||
|
||||
def grammar(self, ast):
|
||||
return model.Grammar(
|
||||
self.name,
|
||||
[
|
||||
r
|
||||
for r in chain(ast.rules, self.synthetic_rules)
|
||||
if r is not None
|
||||
],
|
||||
)
|
||||
|
||||
def rule(self, ast):
|
||||
name = camel2py(ast.name)
|
||||
exp = ast.exp
|
||||
if name[0].isupper():
|
||||
name = name.upper()
|
||||
if isinstance(exp, model.Token):
|
||||
if name in self.token_rules:
|
||||
self.token_rules[
|
||||
name
|
||||
].exp = exp # it is a model._Decorator
|
||||
else:
|
||||
self.token_rules[name] = exp
|
||||
return None
|
||||
elif not ast.fragment and not isinstance(exp, model.Sequence):
|
||||
ref = model.RuleRef(name.lower())
|
||||
if name in self.token_rules:
|
||||
self.token_rules[name].exp = ref
|
||||
else:
|
||||
self.token_rules[name] = ref
|
||||
name = name.lower()
|
||||
|
||||
return model.Rule(ast, name, exp, ast.params, ast.kwparams)
|
||||
|
||||
def alternatives(self, ast):
|
||||
options = [o for o in ast.options if o is not None]
|
||||
if len(options) == 1:
|
||||
return options[0]
|
||||
else:
|
||||
options = [model.Option(o) for o in options]
|
||||
return model.Choice(options)
|
||||
|
||||
def elements(self, ast):
|
||||
elements = [e for e in ast if e is not None]
|
||||
if not elements:
|
||||
return model.Void()
|
||||
elif len(elements) == 1:
|
||||
return elements[0]
|
||||
else:
|
||||
return model.Sequence(AST(sequence=elements))
|
||||
|
||||
def predicate_or_action(self, ast):
|
||||
return None
|
||||
|
||||
def named(self, ast):
|
||||
if ast.force_list:
|
||||
return model.NamedList(ast)
|
||||
else:
|
||||
return model.Named(ast)
|
||||
|
||||
def syntactic_predicate(self, ast):
|
||||
return None
|
||||
|
||||
def optional(self, ast):
|
||||
if isinstance(ast, model.Group | model.Optional | model.Closure):
|
||||
ast = ast.exp
|
||||
return model.Optional(ast)
|
||||
|
||||
def closure(self, ast):
|
||||
if isinstance(ast, model.Group | model.Optional):
|
||||
ast = ast.exp
|
||||
return model.Closure(ast)
|
||||
|
||||
def positive_closure(self, ast):
|
||||
if isinstance(ast, model.Group):
|
||||
ast = ast.exp
|
||||
return model.PositiveClosure(ast)
|
||||
|
||||
def negative(self, ast):
|
||||
neg = model.NegativeLookahead(ast)
|
||||
any = model.Pattern('.')
|
||||
return model.Sequence(AST(sequence=[neg, any]))
|
||||
|
||||
def subexp(self, ast):
|
||||
return model.Group(ast)
|
||||
|
||||
def regexp(self, ast):
|
||||
pattern = ''.join(ast)
|
||||
re.compile(pattern)
|
||||
return model.Pattern(pattern)
|
||||
|
||||
def charset_optional(self, ast):
|
||||
return f'{ast}?'
|
||||
|
||||
def charset_closure(self, ast):
|
||||
return f'{ast}*'
|
||||
|
||||
def charset_positive_closure(self, ast):
|
||||
return f'{ast}+'
|
||||
|
||||
def charset_or(self, ast):
|
||||
return '[{}]'.format(''.join(ast))
|
||||
|
||||
def charset_negative_or(self, ast):
|
||||
return '[^{}]'.format(''.join(ast))
|
||||
|
||||
@staticmethod
|
||||
def escape(s):
|
||||
return ''.join('\\' + c if c in '[]().*+{}^$' else c for c in s)
|
||||
|
||||
def charset_atom(self, ast):
|
||||
return ast
|
||||
|
||||
def charset_char(self, ast):
|
||||
return ast
|
||||
|
||||
def charset_range(self, ast):
|
||||
return f'{ast.first}-{ast.last}'
|
||||
|
||||
def newranges(self, ast):
|
||||
pattern = ''.join(ast)
|
||||
re.compile(pattern)
|
||||
return model.Pattern(pattern)
|
||||
|
||||
def newrange(self, ast):
|
||||
pattern = '[{}]{}'.format(ast.range, ast.repeat or '')
|
||||
re.compile(pattern)
|
||||
return pattern
|
||||
|
||||
def negative_newrange(self, ast):
|
||||
pattern = '[^{}]{}'.format(ast.range, ast.repeat or '')
|
||||
re.compile(pattern)
|
||||
return pattern
|
||||
|
||||
def rule_ref(self, ast):
|
||||
assert ast[0].islower()
|
||||
return model.RuleRef(camel2py(ast))
|
||||
|
||||
def any(self, ast):
|
||||
return model.Pattern(r'\w+|\S+')
|
||||
|
||||
def string(self, ast):
|
||||
text = ast
|
||||
if isinstance(text, list):
|
||||
text = ''.join(text)
|
||||
return model.Token(text)
|
||||
|
||||
def eof(self, ast):
|
||||
return model.EOF()
|
||||
|
||||
def token(self, ast):
|
||||
name = ast.name
|
||||
if ast.value:
|
||||
exp = model.Token(ast.value)
|
||||
self.tokens[name] = exp
|
||||
else:
|
||||
exp = model.Fail()
|
||||
rule = model.Rule(ast, name, exp, [], {})
|
||||
self.synthetic_rules.append(rule)
|
||||
return exp
|
||||
|
||||
def token_ref(self, ast):
|
||||
name = camel2py(ast).upper()
|
||||
|
||||
value = self.tokens.get(name)
|
||||
if value and isinstance(value, model.Model):
|
||||
return value
|
||||
|
||||
if name in self.token_rules:
|
||||
exp = self.token_rules[name]
|
||||
else:
|
||||
exp = model.Decorator(model.RuleRef(name))
|
||||
self.token_rules[name] = exp
|
||||
return exp
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,269 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import dataclasses
|
||||
import re
|
||||
from collections.abc import Callable, MutableMapping
|
||||
from itertools import starmap
|
||||
from typing import Any, NamedTuple
|
||||
|
||||
from .ast import AST
|
||||
from .tokenizing import Tokenizer
|
||||
from .util.misc import cached_re_compile
|
||||
from .util.unicode_characters import C_DERIVE
|
||||
|
||||
|
||||
class UndefinedStr(str):
|
||||
pass
|
||||
|
||||
|
||||
_undefined_str = UndefinedStr('>>undefined<<')
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class ParserConfig:
|
||||
owner: Any = None
|
||||
name: str | None = 'Test'
|
||||
filename: str = ''
|
||||
encoding: str = 'utf-8'
|
||||
|
||||
start: str | None = None # FIXME
|
||||
start_rule: str | None = None # FIXME
|
||||
rule_name: str | None = None # Backward compatibility
|
||||
|
||||
comments_re: re.Pattern | str | None = None
|
||||
eol_comments_re: re.Pattern | str | None = None
|
||||
|
||||
tokenizercls: type[Tokenizer] | None = None # FIXME
|
||||
semantics: type | None = None
|
||||
|
||||
comment_recovery: bool = False
|
||||
memoize_lookaheads: bool = True
|
||||
|
||||
colorize: bool = False
|
||||
trace: bool = False
|
||||
trace_filename: bool = False
|
||||
trace_length: int = 72
|
||||
trace_separator: str = C_DERIVE
|
||||
|
||||
# parser directives
|
||||
grammar: str | None = None
|
||||
left_recursion: bool = True
|
||||
|
||||
comments: str | None = None
|
||||
eol_comments: str | None = None
|
||||
keywords: list[str] | set[str] = dataclasses.field(default_factory=list)
|
||||
|
||||
ignorecase: bool | None = False
|
||||
namechars: str = ''
|
||||
nameguard: bool | None = None # implied by namechars
|
||||
whitespace: str | None = _undefined_str
|
||||
|
||||
parseinfo: bool = False
|
||||
|
||||
def __post_init__(self): # pylint: disable=W0235
|
||||
if self.ignorecase:
|
||||
self.keywords = [k.upper() for k in self.keywords]
|
||||
|
||||
if self.comments_re or self.eol_comments_re:
|
||||
raise AttributeError(
|
||||
"Both `comments_re` and `eol_comments_re` have been removed from parser configurations. " +
|
||||
"Please use `comments` and/or `eol_comments` instead`.",
|
||||
)
|
||||
del self.comments_re
|
||||
del self.eol_comments_re
|
||||
|
||||
if self.comments:
|
||||
cached_re_compile(self.comments)
|
||||
if self.eol_comments:
|
||||
cached_re_compile(self.eol_comments)
|
||||
if self.whitespace:
|
||||
cached_re_compile(self.whitespace)
|
||||
|
||||
@classmethod
|
||||
def new(
|
||||
cls,
|
||||
config: ParserConfig | None = None,
|
||||
owner: Any | None = None,
|
||||
**settings: Any,
|
||||
) -> ParserConfig:
|
||||
result = cls(owner=owner)
|
||||
if config is not None:
|
||||
result = config.replace_config(config)
|
||||
return result.replace(**settings)
|
||||
|
||||
# NOTE:
|
||||
# Using functools.cache directly makes objects of this class unhashable
|
||||
|
||||
def effective_rule_name(self):
|
||||
# note: there are legacy reasons for this mess
|
||||
return self.start_rule or self.rule_name or self.start
|
||||
|
||||
def _find_common(self, **settings: Any) -> MutableMapping[str, Any]:
|
||||
return {
|
||||
name: value
|
||||
for name, value in settings.items()
|
||||
if value is not None and hasattr(self, name)
|
||||
}
|
||||
|
||||
def replace_config(
|
||||
self, other: ParserConfig | None = None,
|
||||
) -> ParserConfig:
|
||||
if other is None:
|
||||
return self
|
||||
elif not isinstance(other, ParserConfig):
|
||||
raise TypeError(f'Unexpected type {type(other).__name__}')
|
||||
else:
|
||||
return self.replace(**vars(other))
|
||||
|
||||
# non-init fields cannot be used as arguments in `replace`, however
|
||||
# they are values returned by `vars` and `dataclass.asdict` so they
|
||||
# must be filtered out.
|
||||
# If the `ParserConfig` dataclass drops these fields, then this filter can be removed
|
||||
def _filter_non_init_fields(self, settings: MutableMapping[str, Any]) -> MutableMapping[str, Any]:
|
||||
for field in [
|
||||
field.name for field in dataclasses.fields(self) if not field.init
|
||||
]:
|
||||
if field in settings:
|
||||
del settings[field]
|
||||
return settings
|
||||
|
||||
def replace(self, **settings: Any) -> ParserConfig:
|
||||
overrides = self._filter_non_init_fields(self._find_common(**settings))
|
||||
result = dataclasses.replace(self, **overrides)
|
||||
if 'grammar' in overrides:
|
||||
result.name = result.grammar
|
||||
return result
|
||||
|
||||
def merge(self, **settings: Any) -> ParserConfig:
|
||||
overrides = self._find_common(**settings)
|
||||
overrides = {
|
||||
name: value
|
||||
for name, value in overrides.items()
|
||||
if getattr(self, name, None) is None
|
||||
}
|
||||
return self.replace(**overrides)
|
||||
|
||||
def asdict(self):
|
||||
# warning: it seems dataclasses.asdict does a deepcopy
|
||||
# result = dataclasses.asdict(self)
|
||||
result = copy.copy(vars(self))
|
||||
result.pop('owner', None)
|
||||
return result
|
||||
|
||||
|
||||
class PosLine(NamedTuple):
|
||||
start: int
|
||||
line: int
|
||||
length: int
|
||||
|
||||
@staticmethod
|
||||
def build_line_cache(lines):
|
||||
cache = []
|
||||
n = 0
|
||||
i = 0
|
||||
for n, line in enumerate(lines):
|
||||
pl = PosLine(i, n, len(line))
|
||||
for _ in line:
|
||||
cache.append(pl) # noqa: PERF401
|
||||
i += len(line)
|
||||
n += 1
|
||||
if lines and lines[-1] and lines[-1][-1] in '\r\n':
|
||||
n += 1
|
||||
cache.append(PosLine(i, n, 0))
|
||||
return cache, n
|
||||
|
||||
|
||||
class LineIndexInfo(NamedTuple):
|
||||
filename: str
|
||||
line: int
|
||||
|
||||
@staticmethod
|
||||
def block_index(name, n):
|
||||
return list(
|
||||
starmap(LineIndexInfo, zip(n * [name], range(n), strict=False)),
|
||||
)
|
||||
|
||||
|
||||
class LineInfo(NamedTuple):
|
||||
filename: str
|
||||
line: int
|
||||
col: int
|
||||
start: int
|
||||
end: int
|
||||
text: int
|
||||
|
||||
|
||||
class CommentInfo(NamedTuple):
|
||||
inline: list
|
||||
eol: list
|
||||
|
||||
@staticmethod
|
||||
def new_comment():
|
||||
return CommentInfo([], [])
|
||||
|
||||
|
||||
class Alert(NamedTuple):
|
||||
level: int = 1
|
||||
message: str = ''
|
||||
|
||||
|
||||
class ParseInfo(NamedTuple):
|
||||
tokenizer: Any
|
||||
rule: str
|
||||
pos: int
|
||||
endpos: int
|
||||
line: int
|
||||
endline: int
|
||||
alerts: list[Alert] = [] # noqa: RUF012
|
||||
|
||||
def text_lines(self):
|
||||
return self.tokenizer.get_lines(self.line, self.endline)
|
||||
|
||||
def line_index(self):
|
||||
return self.tokenizer.line_index(self.line, self.endline)
|
||||
|
||||
@property
|
||||
def buffer(self):
|
||||
return self.tokenizer
|
||||
|
||||
|
||||
class MemoKey(NamedTuple):
|
||||
pos: int
|
||||
rule: str
|
||||
state: Any
|
||||
|
||||
|
||||
class RuleInfo(NamedTuple):
|
||||
name: str
|
||||
impl: Callable
|
||||
is_leftrec: bool
|
||||
is_memoizable: bool
|
||||
is_name: bool
|
||||
params: list
|
||||
kwparams: dict
|
||||
|
||||
def __hash__(self):
|
||||
return hash(self.name)
|
||||
|
||||
def __eq__(self, other):
|
||||
if isinstance(other, RuleInfo):
|
||||
return self.name == other.name
|
||||
return False
|
||||
|
||||
def __ne__(self, other):
|
||||
return not self.__eq__(other)
|
||||
|
||||
|
||||
class RuleResult(NamedTuple):
|
||||
node: Any
|
||||
newpos: int
|
||||
newstate: Any
|
||||
|
||||
|
||||
@dataclasses.dataclass(slots=True)
|
||||
class ParseState:
|
||||
pos: int = 0
|
||||
ast: AST = dataclasses.field(default_factory=AST)
|
||||
cst: Any = None
|
||||
alerts: list[Alert] = dataclasses.field(default_factory=list)
|
||||
@@ -0,0 +1,198 @@
|
||||
# ruff: noqa: PLW2901
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import defaultdict
|
||||
from collections.abc import Callable
|
||||
|
||||
from . import grammars
|
||||
|
||||
# Based on https://github.com/ncellar/autumn_v1/
|
||||
|
||||
|
||||
# Returns the correct Rule instance for a RuleRef
|
||||
def follow(node, rule_dict):
|
||||
if isinstance(node, grammars.RuleRef):
|
||||
return rule_dict.get(node.name, node)
|
||||
else:
|
||||
return node
|
||||
|
||||
|
||||
class Nullable:
|
||||
def __init__(self, children, resolved=False, nullable=False):
|
||||
self.resolved = resolved
|
||||
self.nullable = nullable
|
||||
self.children = children
|
||||
|
||||
def resolve(self, node, rule_dict):
|
||||
pass
|
||||
|
||||
def resolve_with(self, n):
|
||||
self.resolved = True
|
||||
self.nullable = n
|
||||
self.children = None # No longer needed
|
||||
|
||||
all: Callable
|
||||
any: Callable
|
||||
of: Callable
|
||||
no: Callable
|
||||
yes: Callable
|
||||
|
||||
|
||||
class _All(Nullable):
|
||||
def resolve(self, node, rule_dict):
|
||||
unresolved = []
|
||||
for c in self.children:
|
||||
c = follow(c, rule_dict)
|
||||
n = c._nullability
|
||||
if n.resolved:
|
||||
if not n.nullable:
|
||||
# Not nullable if any is not nullable
|
||||
self.resolve_with(False)
|
||||
return
|
||||
else:
|
||||
unresolved.append(c)
|
||||
if not unresolved:
|
||||
# Nullable if all are nullable
|
||||
self.resolve_with(True)
|
||||
else:
|
||||
# Otherwise still unresolved
|
||||
self.children = unresolved
|
||||
|
||||
|
||||
class _Any(Nullable):
|
||||
def resolve(self, node, rule_dict):
|
||||
# Inverse of All
|
||||
unresolved = []
|
||||
for c in self.children:
|
||||
c = follow(c, rule_dict)
|
||||
n = c._nullability
|
||||
if n.resolved:
|
||||
if n.nullable:
|
||||
self.resolve_with(True)
|
||||
return
|
||||
else:
|
||||
unresolved.append(c)
|
||||
if not unresolved:
|
||||
self.resolve_with(False)
|
||||
else:
|
||||
self.children = unresolved
|
||||
|
||||
|
||||
class _Single(Nullable):
|
||||
def resolve(self, node, rule_dict):
|
||||
n = follow(self.children[0], rule_dict)._nullability
|
||||
if not n.resolved:
|
||||
return
|
||||
self.resolve_with(n.nullable)
|
||||
|
||||
|
||||
Nullable.all = _All # Nullable if all children are nullable
|
||||
Nullable.any = _Any # Nullable if one child is nullable
|
||||
Nullable.of = lambda child: _Single(
|
||||
[child],
|
||||
) # Nullable if the only child is nullable
|
||||
Nullable.no = lambda: Nullable(None, True, False) # Not nullable
|
||||
Nullable.yes = lambda: Nullable(None, True, True) # Nullable
|
||||
|
||||
|
||||
def resolve_nullability(grammar, rule_dict):
|
||||
dependants = defaultdict(list)
|
||||
visited = set() # To prevent infinite recursion
|
||||
|
||||
def walk(model): # TODO Write a walker for this?
|
||||
if model in visited:
|
||||
return
|
||||
visited.add(model)
|
||||
|
||||
for child in model.children_list():
|
||||
child = follow(child, rule_dict)
|
||||
walk(child)
|
||||
|
||||
resolve(model)
|
||||
|
||||
def resolve(model):
|
||||
nullability = model._nullability
|
||||
if not nullability.resolved:
|
||||
nullability.resolve(model, rule_dict)
|
||||
if nullability.resolved:
|
||||
for dependant in dependants[model]:
|
||||
n = dependant._nullability
|
||||
if not n.resolved:
|
||||
resolve(
|
||||
dependant,
|
||||
) # Resolve nodes that depend on this one
|
||||
else:
|
||||
for n in nullability.children:
|
||||
dependants[n].append(model)
|
||||
|
||||
walk(grammar)
|
||||
|
||||
|
||||
# This breaks left recursive cycles by tagging
|
||||
# left recursive rules
|
||||
|
||||
|
||||
def find_left_recursion(grammar):
|
||||
rule_dict = {
|
||||
rule.name: rule for rule in grammar.rules
|
||||
} # Required to resolve rule references
|
||||
|
||||
# First we need to resolve nullable rules
|
||||
resolve_nullability(grammar, rule_dict)
|
||||
|
||||
# Traversable state
|
||||
FIRST = 0
|
||||
CUTOFF = 1
|
||||
VISITED = 2
|
||||
|
||||
state = defaultdict(lambda: FIRST)
|
||||
stack_depth = [0] # nonlocal workaround 2.7
|
||||
stack_positions = {}
|
||||
lr_stack_positions = [-1]
|
||||
|
||||
def walk(model):
|
||||
if state[model] == FIRST:
|
||||
state[model] = CUTOFF
|
||||
else:
|
||||
return
|
||||
|
||||
# beforeNode
|
||||
leftrec = isinstance(model, grammars.Rule) and model.is_leftrec
|
||||
if leftrec:
|
||||
lr_stack_positions.append(stack_depth[0])
|
||||
|
||||
stack_positions[model] = stack_depth[0]
|
||||
stack_depth[0] += 1
|
||||
|
||||
for child in model.at_same_pos(rule_dict):
|
||||
child = follow(child, rule_dict)
|
||||
walk(child)
|
||||
# afterEdge
|
||||
if (
|
||||
state[child] == CUTOFF
|
||||
and stack_positions[child] > lr_stack_positions[-1]
|
||||
):
|
||||
# turn off memoization for all rules that were involved in this cycle
|
||||
for rule in stack_positions:
|
||||
if isinstance(rule, grammars.Rule):
|
||||
rule.is_memoizable = False
|
||||
|
||||
child.is_leftrec = True
|
||||
|
||||
# afterNode
|
||||
if leftrec:
|
||||
lr_stack_positions.pop()
|
||||
|
||||
del stack_positions[model]
|
||||
stack_depth[0] -= 1
|
||||
|
||||
state[model] = VISITED
|
||||
|
||||
for rule in grammar.children_list():
|
||||
walk(rule)
|
||||
|
||||
# print()
|
||||
# for rule in grammar.children_list():
|
||||
# print(rule)
|
||||
# if rule.is_leftrec: print("-> Leftrec")
|
||||
# if rule.is_nullable(): print("-> Nullable")
|
||||
@@ -0,0 +1,65 @@
|
||||
import io
|
||||
from contextlib import contextmanager
|
||||
|
||||
from ..util import trim
|
||||
|
||||
|
||||
class IndentPrintMixin:
|
||||
def __init__(self, default_indent: int = 4):
|
||||
self.default_indent = default_indent
|
||||
self.indent_stack: list[int] = [0]
|
||||
self.output_stream = io.StringIO()
|
||||
|
||||
def printed_text(self):
|
||||
return self.output_stream.getvalue()
|
||||
|
||||
def print(self, *args, **kwargs):
|
||||
args = [trim(str(arg)) for arg in args if arg is not None]
|
||||
lines = self.as_printed_lines(*args, **kwargs)
|
||||
self._do_print_lines(lines, **kwargs)
|
||||
|
||||
def as_printed(self, *args, **kwargs):
|
||||
lines = self.as_printed_lines(*args, **kwargs)
|
||||
return '\n'.join(lines)
|
||||
|
||||
def as_printed_lines(self, *args, **kwargs):
|
||||
text = self.io_print(*args, **kwargs)
|
||||
return self.indented_lines(text)
|
||||
|
||||
@contextmanager
|
||||
def indent(self, amount: int | None = None):
|
||||
assert amount is None or amount >= 0
|
||||
if amount is None:
|
||||
amount = self.default_indent
|
||||
|
||||
self.indent_stack.append(amount + self.indent_stack[-1])
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self.indent_stack.pop()
|
||||
|
||||
@property
|
||||
def current_indentation(self):
|
||||
return ' ' * self.indent_stack[-1]
|
||||
|
||||
@staticmethod
|
||||
def io_print(*args, **kwargs):
|
||||
kwargs.pop('file', None) # do not allow redirection of output
|
||||
with io.StringIO() as output:
|
||||
print(*args, file=output, **kwargs)
|
||||
return output.getvalue()
|
||||
|
||||
def _do_print_lines(self, lines: list[str] | None = None, **kwargs):
|
||||
if not lines:
|
||||
print(file=self.output_stream, **kwargs)
|
||||
return
|
||||
|
||||
for line in lines:
|
||||
print(line, file=self.output_stream, **kwargs)
|
||||
|
||||
def indented_lines(self, text):
|
||||
text = trim(text)
|
||||
return [
|
||||
(self.current_indentation + line).rstrip()
|
||||
for line in text.splitlines(keepends=False)
|
||||
]
|
||||
@@ -0,0 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from .ast import AST # noqa: F401
|
||||
from .objectmodel import Node # noqa: F401
|
||||
from .objectmodel import Node as ParseModel # noqa: F401
|
||||
from .semantics import ModelBuilderSemantics # noqa: F401
|
||||
from .walkers import DepthFirstWalker, NodeWalker # noqa: F401 # noqa: F401
|
||||
@@ -0,0 +1,8 @@
|
||||
from ..objectmodel import ParseModel
|
||||
from .python import PythonCodeGenerator
|
||||
|
||||
|
||||
def codegen(model: ParseModel, parser_name: str = '') -> str:
|
||||
generator = PythonCodeGenerator(parser_name=parser_name)
|
||||
generator.walk(model)
|
||||
return generator.printed_text()
|
||||
@@ -0,0 +1,149 @@
|
||||
import builtins
|
||||
from collections import namedtuple
|
||||
|
||||
from .. import grammars, objectmodel
|
||||
from ..mixins.indent import IndentPrintMixin
|
||||
from ..util import compress_seq, safe_name
|
||||
from ..util.misc import topsort
|
||||
|
||||
HEADER = """\
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# WARNING: CAVEAT UTILITOR
|
||||
#
|
||||
# This file was automatically generated by TatSu.
|
||||
#
|
||||
# https://pypi.python.org/pypi/tatsu/
|
||||
#
|
||||
# Any changes you make to it will be overwritten the next time
|
||||
# the file is generated.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from dataclasses import dataclass
|
||||
|
||||
from tatsu.semantics import ModelBuilderSemantics
|
||||
{base_type_import}
|
||||
|
||||
|
||||
class {name}ModelBuilderSemantics(ModelBuilderSemantics):
|
||||
def __init__(self, context=None, types=None):
|
||||
types = [
|
||||
t for t in globals().values()
|
||||
if type(t) is type and issubclass(t, ModelBase)
|
||||
] + (types or [])
|
||||
super().__init__(context=context, types=types)
|
||||
"""
|
||||
|
||||
|
||||
BaseClassSpec = namedtuple('BaseClassSpec', ['class_name', 'base'])
|
||||
|
||||
|
||||
def modelgen(model: grammars.Grammar, name: str = '', base_type: type | None = objectmodel.Node) -> str:
|
||||
base_type = base_type or objectmodel.Node
|
||||
generator = PythonModelGenerator(name=name, base_type=base_type)
|
||||
return generator.generate_model(model)
|
||||
|
||||
|
||||
class PythonModelGenerator(IndentPrintMixin):
|
||||
|
||||
def __init__(self, name: str = '', base_type: type = objectmodel.Node):
|
||||
super().__init__()
|
||||
self.base_type = base_type
|
||||
self.name = name or None
|
||||
|
||||
def generate_model(self, grammar: grammars.Grammar):
|
||||
base_type = self.base_type
|
||||
base_type_name = base_type.__name__.split('.')[-1]
|
||||
base_type_import = f"from {base_type.__module__} import {base_type_name}"
|
||||
|
||||
self.name = self.name or grammar.name
|
||||
self.print(
|
||||
HEADER.format(
|
||||
name=self.name,
|
||||
base_type=self.base_type.__name__,
|
||||
base_type_import=base_type_import,
|
||||
),
|
||||
)
|
||||
|
||||
rule_index = {rule.name: rule for rule in grammar.rules}
|
||||
rule_specs = {
|
||||
rule.name: self._base_class_specs(rule)
|
||||
for rule in grammar.rules
|
||||
}
|
||||
rule_specs = {name: specs for name, specs in rule_specs.items() if specs}
|
||||
|
||||
specs_by_name = {
|
||||
s.class_name: s.base
|
||||
for specs in rule_specs.values()
|
||||
for s in specs
|
||||
}
|
||||
base = self._model_base_name()
|
||||
specs_by_name[base] = base_type_name
|
||||
|
||||
all_specs = {
|
||||
(s.class_name, s.base)
|
||||
for specs in rule_specs.values()
|
||||
for s in specs
|
||||
}
|
||||
model_names = topsort(reversed(specs_by_name), all_specs)
|
||||
|
||||
model_to_rule = {
|
||||
rule_specs[name][0].class_name: rule
|
||||
for name, rule in rule_index.items()
|
||||
if name in rule_specs
|
||||
}
|
||||
|
||||
for model_name in model_names:
|
||||
if model_name in vars(builtins):
|
||||
continue
|
||||
if rule := model_to_rule.get(model_name):
|
||||
self._gen_rule_class(rule, rule_specs[rule.name])
|
||||
else:
|
||||
self._gen_base_class(model_name, specs_by_name.get(model_name))
|
||||
|
||||
return self.printed_text()
|
||||
|
||||
@staticmethod
|
||||
def _model_base_name():
|
||||
return 'ModelBase'
|
||||
|
||||
def _gen_base_class(self, class_name: str, base: str | None):
|
||||
self.print()
|
||||
self.print()
|
||||
self.print('@dataclass(eq=False)')
|
||||
if base:
|
||||
self.print(f'class {class_name}({base}):')
|
||||
else:
|
||||
# FIXME: this cannot happen as base_type is the final base
|
||||
self.print(f'class {class_name}:')
|
||||
with self.indent():
|
||||
self.print('pass')
|
||||
|
||||
def _gen_rule_class(self, rule: grammars.Rule, specs: list[BaseClassSpec]):
|
||||
if not specs:
|
||||
return
|
||||
spec = specs[0]
|
||||
arguments = sorted({safe_name(d) for d, _ in compress_seq(rule.defines())})
|
||||
|
||||
self.print()
|
||||
self.print()
|
||||
self.print('@dataclass(eq=False)')
|
||||
self.print(f'class {spec.class_name}({spec.base}):')
|
||||
with self.indent():
|
||||
if not arguments:
|
||||
self.print('pass')
|
||||
for arg in arguments:
|
||||
self.print(f'{arg}: Any = None')
|
||||
|
||||
def _base_class_specs(self, rule: grammars.Rule) -> list[BaseClassSpec]:
|
||||
if not rule.params or not isinstance(rule.params[0], str):
|
||||
return []
|
||||
spec = rule.params[0].split('::')
|
||||
base = [self._model_base_name()]
|
||||
class_names = [safe_name(n) for n in spec] + base
|
||||
return [
|
||||
BaseClassSpec(class_name, class_names[i + 1])
|
||||
for i, class_name in enumerate(class_names[:-1])
|
||||
]
|
||||
@@ -0,0 +1,392 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import itertools
|
||||
import textwrap
|
||||
from collections.abc import Iterator
|
||||
from typing import Any
|
||||
|
||||
from .. import grammars
|
||||
from ..collections import OrderedSet as oset
|
||||
from ..exceptions import CodegenError
|
||||
from ..infos import UndefinedStr
|
||||
from ..mixins.indent import IndentPrintMixin
|
||||
from ..util import compress_seq, safe_name
|
||||
from ..walkers import NodeWalker
|
||||
|
||||
HEADER = """\
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# WARNING: CAVEAT UTILITOR
|
||||
#
|
||||
# This file was automatically generated by TatSu.
|
||||
#
|
||||
# https://pypi.python.org/pypi/tatsu/
|
||||
#
|
||||
# Any changes you make to it will be overwritten the next time
|
||||
# the file is generated.
|
||||
|
||||
# ruff: noqa: C405, COM812, I001, F401, PLR1702, PLC2801, SIM117
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from tatsu.buffering import Buffer
|
||||
from tatsu.parsing import Parser
|
||||
from tatsu.parsing import tatsumasu
|
||||
from tatsu.parsing import leftrec, nomemo, isname
|
||||
from tatsu.infos import ParserConfig
|
||||
from tatsu.util import re, generic_main
|
||||
"""
|
||||
|
||||
FOOTER = """\
|
||||
def main(filename, **kwargs):
|
||||
if not filename or filename == '-':
|
||||
text = sys.stdin.read()
|
||||
else:
|
||||
text = Path(filename).read_text()
|
||||
parser = {name}Parser()
|
||||
return parser.parse(
|
||||
text,
|
||||
filename=filename,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
import json
|
||||
from tatsu.util import asjson
|
||||
|
||||
ast = generic_main(main, {name}Parser, name='{name}')
|
||||
data = asjson(ast)
|
||||
print(json.dumps(data, indent=2))
|
||||
"""
|
||||
|
||||
|
||||
class PythonCodeGenerator(IndentPrintMixin, NodeWalker):
|
||||
_counter: Iterator[int] = itertools.count()
|
||||
|
||||
def __init__(self, parser_name: str = ''):
|
||||
super().__init__()
|
||||
self.parser_name = parser_name
|
||||
|
||||
@classmethod
|
||||
def _next_n(cls):
|
||||
return next(cls._counter)
|
||||
|
||||
@classmethod
|
||||
def _reset_counter(cls):
|
||||
cls._counter = itertools.count()
|
||||
|
||||
def walk_default(self, node: Any):
|
||||
return node
|
||||
|
||||
def walk_Grammar(self, grammar: grammars.Grammar):
|
||||
parser_name = self.parser_name or grammar.name
|
||||
self.print(HEADER)
|
||||
self.print()
|
||||
self.print()
|
||||
|
||||
self._gen_keywords(grammar)
|
||||
self._gen_buffering(grammar, parser_name)
|
||||
self._gen_parsing(grammar, parser_name)
|
||||
|
||||
self.print(FOOTER.format(name=parser_name))
|
||||
|
||||
def walk_Rule(self, rule: grammars.Rule):
|
||||
def param_repr(p):
|
||||
if isinstance(p, int | float):
|
||||
return str(p)
|
||||
else:
|
||||
return repr(p.split('::')[0])
|
||||
|
||||
self._reset_counter()
|
||||
params = kwparams = ''
|
||||
if rule.params:
|
||||
params = ', '.join(
|
||||
param_repr(self.walk(p)) for p in rule.params
|
||||
)
|
||||
if rule.kwparams:
|
||||
kwparams = ', '.join(
|
||||
f'{k}={param_repr(self.walk(v))}'
|
||||
for k, v in rule.kwparams.items()
|
||||
)
|
||||
|
||||
if params and kwparams:
|
||||
params = params + ', ' + kwparams
|
||||
elif kwparams:
|
||||
params = kwparams
|
||||
|
||||
leftrec = '\n@leftrec' if rule.is_leftrec else ''
|
||||
nomemo = (
|
||||
'\n@nomemo'
|
||||
if not rule.is_memoizable and not leftrec
|
||||
else ''
|
||||
)
|
||||
isname = '\n@isname' if rule.is_name else ''
|
||||
|
||||
self.print(
|
||||
f"""
|
||||
@tatsumasu({params})\
|
||||
{leftrec}\
|
||||
{nomemo}\
|
||||
{isname}\
|
||||
\ndef _{rule.name}_(self):
|
||||
""",
|
||||
)
|
||||
with self.indent():
|
||||
self.print(self.walk(rule.exp))
|
||||
|
||||
def walk_BasedRule(self, rule: grammars.BasedRule):
|
||||
# FIXME: the following override is to not alter the previous codegen
|
||||
rule.exp = rule.rhs
|
||||
self.walk_Rule(rule)
|
||||
|
||||
def walk_RuleRef(self, ref: grammars.RuleRef):
|
||||
self.print(f'self._{ref.name}_()')
|
||||
|
||||
def walk_RuleInclude(self, include: grammars.RuleInclude):
|
||||
self.walk(include.rule.exp)
|
||||
|
||||
def walk_Void(self, void: grammars.Void):
|
||||
self.print('self._void()')
|
||||
|
||||
def walk_Any(self, any: grammars.Any):
|
||||
self.print('self._any()')
|
||||
|
||||
def walk_Fail(self, fail: grammars.Fail):
|
||||
self.print('self._fail()')
|
||||
|
||||
def walk_Cut(self, cut: grammars.Cut):
|
||||
self.print('self._cut()')
|
||||
|
||||
def walk_Comment(self, comment: grammars.Comment):
|
||||
lines = '\n'.join(f'# {c!s}' for c in comment.comment.splitlines())
|
||||
self.print(f'\n{lines}\n')
|
||||
|
||||
def walk_EOLComment(self, comment: grammars.EOLComment):
|
||||
self.walk_Comment(comment)
|
||||
|
||||
def walk_EOF(self, eof: grammars.EOF):
|
||||
self.print('self._check_eof()')
|
||||
|
||||
def walk_Group(self, group: grammars.Group):
|
||||
self.print('with self._group():')
|
||||
with self.indent():
|
||||
self.walk(group.exp)
|
||||
|
||||
def walk_Token(self, token: grammars.Token):
|
||||
self.print(f'self._token({token.token!r})')
|
||||
|
||||
def walk_Constant(self, constant: grammars.Constant):
|
||||
self.print(f'self._constant({constant.literal!r})')
|
||||
|
||||
def walk_Alert(self, alert: grammars.Alert):
|
||||
self.print(f'self._alert({alert.literal!r}, {alert.level})')
|
||||
|
||||
def walk_Pattern(self, pattern: grammars.Pattern):
|
||||
self.print(f'self._pattern({pattern.pattern!r})')
|
||||
|
||||
def walk_Lookahead(self, lookahead: grammars.Lookahead):
|
||||
self.print('with self._if():')
|
||||
with self.indent():
|
||||
self.walk(lookahead.exp)
|
||||
|
||||
def walk_NegativeLookahead(self, lookahead: grammars.NegativeLookahead):
|
||||
self.print('with self._ifnot():')
|
||||
with self.indent():
|
||||
self.walk(lookahead.exp)
|
||||
|
||||
def walk_Sequence(self, seq: grammars.Sequence):
|
||||
self.walk(seq.sequence)
|
||||
self._gen_defines_declaration(seq)
|
||||
|
||||
def walk_Choice(self, choice: grammars.Choice):
|
||||
if len(choice.options) == 1:
|
||||
self.walk(choice.options[0])
|
||||
return
|
||||
|
||||
firstset = choice.lookahead_str()
|
||||
if firstset:
|
||||
msglines = textwrap.wrap(firstset, width=40)
|
||||
error = ['expecting one of: ', *msglines]
|
||||
else:
|
||||
error = ['no available options']
|
||||
errors = '\n'.join(repr(e) for e in error)
|
||||
|
||||
self.print('with self._choice():')
|
||||
with self.indent():
|
||||
self.walk(choice.options)
|
||||
self.print('self._error(')
|
||||
with self.indent():
|
||||
self.print(errors)
|
||||
self.print(')')
|
||||
|
||||
def walk_Option(self, option: grammars.Option):
|
||||
self.print('with self._option():')
|
||||
with self.indent():
|
||||
self.walk(option.exp)
|
||||
|
||||
def walk_Optional(self, optional: grammars.Optional):
|
||||
self.print('with self._optional():')
|
||||
with self.indent():
|
||||
self.walk(optional.exp)
|
||||
|
||||
def walk_EmptyClosure(self, closure: grammars.EmptyClosure):
|
||||
self.print('self._empty_closure()')
|
||||
|
||||
def walk_Closure(self, closure: grammars.Closure):
|
||||
n = self._gen_block(closure.exp)
|
||||
self.print(f'self._closure(block{n})')
|
||||
|
||||
def walk_PositiveClosure(self, closure: grammars.PositiveClosure):
|
||||
n = self._gen_block(closure.exp)
|
||||
self.print(f'self._positive_closure(block{n})')
|
||||
|
||||
def walk_Join(self, join: grammars.Join):
|
||||
m = self._gen_block(join.sep, name='sep')
|
||||
n = self._gen_block(join.exp)
|
||||
self.print(f'self._join(block{n}, sep{m})')
|
||||
|
||||
def walk_PositiveJoin(self, join: grammars.PositiveJoin):
|
||||
m = self._gen_block(join.sep, name='sep')
|
||||
n = self._gen_block(join.exp)
|
||||
self.print(f'self._positive_join(block{n}, sep{m})')
|
||||
|
||||
def walk_LeftJoin(self, join: grammars.LeftJoin):
|
||||
m = self._gen_block(join.sep, name='sep')
|
||||
n = self._gen_block(join.exp)
|
||||
self.print(f'self._left_join(block{n}, sep{m})')
|
||||
|
||||
def walk_RightJoin(self, join: grammars.RightJoin):
|
||||
m = self._gen_block(join.sep, name='sep')
|
||||
n = self._gen_block(join.exp)
|
||||
self.print(f'self._right_join(block{n}, sep{m})')
|
||||
|
||||
def walk_Gather(self, gather: grammars.Gather):
|
||||
m = self._gen_block(gather.sep, name='sep')
|
||||
n = self._gen_block(gather.exp)
|
||||
self.print(f'self._gather(block{n}, sep{m})')
|
||||
|
||||
def walk_PositiveGather(self, gather: grammars.PositiveGather):
|
||||
m = self._gen_block(gather.sep, name='sep')
|
||||
n = self._gen_block(gather.exp)
|
||||
self.print(f'self._positive_gather(block{n}, sep{m})')
|
||||
|
||||
def walk_SkipTo(self, skipto: grammars.SkipTo):
|
||||
n = self._gen_block(skipto.exp)
|
||||
self.print(f'self._skip_to(block{n})')
|
||||
|
||||
def walk_Named(self, named: grammars.Named):
|
||||
self.walk(named.exp)
|
||||
self.print(f"self.name_last_node('{named.name}')")
|
||||
|
||||
def walk_NamedList(self, named: grammars.Named):
|
||||
self.walk(named.exp)
|
||||
self.print(f"self.add_last_node_to_name('{named.name}')")
|
||||
|
||||
def walk_Override(self, override: grammars.Override):
|
||||
self.walk_Named(override)
|
||||
|
||||
def walk_OverrideList(self, override: grammars.OverrideList):
|
||||
self.walk_NamedList(override)
|
||||
|
||||
def walk_Special(self, special: grammars.Special):
|
||||
pass
|
||||
|
||||
def _gen_keywords(self, grammar: grammars.Grammar):
|
||||
keywords = [str(k) for k in grammar.keywords if k is not None]
|
||||
if not keywords:
|
||||
self.print('KEYWORDS: set[str] = set()')
|
||||
else:
|
||||
keywords_str = '\n'.join(f' {k!r},' for k in keywords)
|
||||
keywords_str = f'{{\n{keywords_str}\n}}'
|
||||
self.print(f'KEYWORDS: set[str] = {keywords_str}')
|
||||
|
||||
self.print()
|
||||
self.print()
|
||||
|
||||
def _gen_init(self, grammar: grammars.Grammar):
|
||||
start = grammar.config.start or grammar.rules[0].name
|
||||
|
||||
whitespace = grammar.config.whitespace
|
||||
if isinstance(whitespace, UndefinedStr): # the default value
|
||||
whitespace = None
|
||||
whitespace = repr(whitespace)
|
||||
|
||||
self.print(
|
||||
f'''
|
||||
config = ParserConfig.new(
|
||||
config,
|
||||
owner=self,
|
||||
whitespace={whitespace},
|
||||
nameguard={grammar.config.nameguard},
|
||||
ignorecase={grammar.config.ignorecase},
|
||||
namechars={grammar.config.namechars!r},
|
||||
parseinfo={grammar.config.parseinfo},
|
||||
comments={grammar.config.comments!r},
|
||||
eol_comments={grammar.config.eol_comments!r},
|
||||
keywords=KEYWORDS,
|
||||
start={start!r},
|
||||
)
|
||||
config = config.replace(**settings)
|
||||
''',
|
||||
)
|
||||
self.print()
|
||||
|
||||
def _gen_buffering(self, grammar: grammars.Grammar, parser_name: str):
|
||||
self.print(f'class {parser_name}Buffer(Buffer):')
|
||||
|
||||
with self.indent():
|
||||
self.print('def __init__(self, text, /, config: ParserConfig | None = None, **settings):')
|
||||
with self.indent():
|
||||
self._gen_init(grammar)
|
||||
self.print('super().__init__(text, config=config)')
|
||||
self.print()
|
||||
|
||||
def _gen_parsing(self, grammar: grammars.Grammar, parser_name: str):
|
||||
self.print()
|
||||
self.print(f'class {parser_name}Parser(Parser):')
|
||||
with self.indent():
|
||||
self.print('def __init__(self, /, config: ParserConfig | None = None, **settings):')
|
||||
with self.indent():
|
||||
self._gen_init(grammar)
|
||||
self.print('super().__init__(config=config)')
|
||||
self.print()
|
||||
self.walk(grammar.rules)
|
||||
self.print()
|
||||
|
||||
def _gen_defines_declaration(self, node: grammars.Model):
|
||||
defines = compress_seq(node.defines())
|
||||
ldefs = oset(safe_name(d) for d, value in defines if value)
|
||||
sdefs = oset(
|
||||
safe_name(d)
|
||||
for d, value in defines
|
||||
if not value and d not in ldefs
|
||||
)
|
||||
|
||||
if not (sdefs or ldefs):
|
||||
return
|
||||
|
||||
sdefs_str = ', '.join(sorted(repr(d) for d in sdefs))
|
||||
ldefs_str = ', '.join(sorted(repr(d) for d in ldefs))
|
||||
|
||||
if not ldefs:
|
||||
self.print(f'self._define([{sdefs_str}], [{ldefs_str}])')
|
||||
else:
|
||||
self.print('self._define(')
|
||||
with self.indent():
|
||||
self.print(f'[{sdefs_str}],')
|
||||
self.print(f'[{ldefs_str}],')
|
||||
self.print(')')
|
||||
|
||||
def _gen_block(self, exp: grammars.Model, name='block'):
|
||||
if () in exp.lookahead():
|
||||
raise CodegenError(f'{exp} may repeat empty sequence')
|
||||
|
||||
n = self._next_n()
|
||||
self.print()
|
||||
self.print(f'def {name}{n}():')
|
||||
with self.indent():
|
||||
self.walk(exp)
|
||||
|
||||
return n
|
||||
@@ -0,0 +1,203 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import weakref
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from .ast import AST
|
||||
from .infos import CommentInfo, ParseInfo
|
||||
from .util import AsJSONMixin, asjson, asjsons
|
||||
|
||||
BASE_CLASS_TOKEN = '::' # noqa: S105
|
||||
|
||||
|
||||
@dataclass(eq=False)
|
||||
class Node(AsJSONMixin):
|
||||
_parent: Node | None = None
|
||||
_children: list[Node] | None = None
|
||||
ast: AST | None = None
|
||||
ctx: Any = None
|
||||
parseinfo: ParseInfo | None = None
|
||||
|
||||
def __init__(self, ast=None, **attributes):
|
||||
super().__init__()
|
||||
if isinstance(ast, dict):
|
||||
ast = AST(ast)
|
||||
self.ast = ast
|
||||
|
||||
for name, value in attributes.items():
|
||||
setattr(self, name, value)
|
||||
|
||||
self.__post_init__()
|
||||
|
||||
def __post_init__(self):
|
||||
ast = self.ast
|
||||
|
||||
if not self.parseinfo and isinstance(ast, AST):
|
||||
self.parseinfo = ast.parseinfo
|
||||
|
||||
if not isinstance(ast, Mapping):
|
||||
return
|
||||
|
||||
for name in set(ast) - {'parseinfo'}:
|
||||
try:
|
||||
setattr(self, name, ast[name])
|
||||
except AttributeError as e:
|
||||
raise AttributeError(f"'{name}' is a reserved name") from e
|
||||
|
||||
self._children = self.children_list()
|
||||
if self.parseinfo is None:
|
||||
del self.parseinfo
|
||||
|
||||
@property
|
||||
def parent(self):
|
||||
return self._parent
|
||||
|
||||
@property
|
||||
def line(self):
|
||||
if self.parseinfo:
|
||||
return self.parseinfo.line
|
||||
return None
|
||||
|
||||
@property
|
||||
def endline(self):
|
||||
if self.parseinfo:
|
||||
return self.parseinfo.endline
|
||||
return None
|
||||
|
||||
def text_lines(self):
|
||||
if self.parseinfo:
|
||||
return self.parseinfo.text_lines()
|
||||
return None
|
||||
|
||||
def line_index(self):
|
||||
return self.parseinfo.line_index()
|
||||
|
||||
@property
|
||||
def col(self):
|
||||
return self.line_info.col if self.line_info else None
|
||||
|
||||
@property
|
||||
def context(self):
|
||||
return self.ctx
|
||||
|
||||
@property
|
||||
def line_info(self):
|
||||
if self.parseinfo:
|
||||
return self.parseinfo.tokenizer.line_info(self.parseinfo.pos)
|
||||
return None
|
||||
|
||||
@property
|
||||
def text(self):
|
||||
if not self.parseinfo:
|
||||
return ''
|
||||
text = self.parseinfo.tokenizer.text
|
||||
return text[self.parseinfo.pos: self.parseinfo.endpos]
|
||||
|
||||
@property
|
||||
def comments(self):
|
||||
if self.parseinfo:
|
||||
return self.parseinfo.tokenizer.comments(self.parseinfo.pos)
|
||||
return CommentInfo([], [])
|
||||
|
||||
@property
|
||||
def _deref(self):
|
||||
# use this to get the actual object over weakref instances
|
||||
return self
|
||||
|
||||
def _find_children(self):
|
||||
def with_parent(node):
|
||||
node._parent = weakref.proxy(self)
|
||||
return node
|
||||
|
||||
def children_of(child):
|
||||
if isinstance(child, weakref.ReferenceType | weakref.ProxyType):
|
||||
return
|
||||
elif isinstance(child, Node):
|
||||
yield with_parent(child)
|
||||
elif isinstance(child, Mapping):
|
||||
for name, value in child.items():
|
||||
if name.startswith('_'):
|
||||
continue
|
||||
yield from children_of(value)
|
||||
elif isinstance(child, list | tuple):
|
||||
yield from (
|
||||
with_parent(c) for c in child if isinstance(c, Node)
|
||||
)
|
||||
|
||||
children = list(self._pubdict().items())
|
||||
if not children:
|
||||
yield from children_of(self.ast)
|
||||
else:
|
||||
for name, child in children:
|
||||
if name.startswith('_'):
|
||||
continue
|
||||
yield from children_of(child)
|
||||
|
||||
def children_list(self):
|
||||
if self._children is not None:
|
||||
return self._children
|
||||
return list(self._find_children())
|
||||
|
||||
def children_set(self):
|
||||
return set(self.children_list())
|
||||
|
||||
def children(self):
|
||||
return self.children_list()
|
||||
|
||||
def asjson(self):
|
||||
return asjson(self)
|
||||
|
||||
def _pubdict(self):
|
||||
return {
|
||||
name: value
|
||||
for name, value in super()._pubdict().items()
|
||||
if name not in {'ast', 'ctx', 'parent', 'parseinfo'}
|
||||
}
|
||||
|
||||
def __str__(self):
|
||||
return asjsons(self)
|
||||
|
||||
def __hash__(self):
|
||||
if self.ast is not None:
|
||||
if isinstance(self.ast, list):
|
||||
return hash(tuple(self.ast))
|
||||
elif isinstance(self.ast, dict):
|
||||
return hash(AST(self.ast))
|
||||
else:
|
||||
return hash(self.ast)
|
||||
else:
|
||||
return id(self)
|
||||
|
||||
def __eq__(self, other):
|
||||
if id(self) == id(other):
|
||||
return True
|
||||
elif self.ast is None:
|
||||
return False
|
||||
elif not getattr(other, 'ast', None):
|
||||
return False
|
||||
else:
|
||||
return self.ast == other.ast
|
||||
|
||||
def _nonrefdict(self):
|
||||
return {
|
||||
name: value
|
||||
for name, value in vars(self).items()
|
||||
if (
|
||||
name not in {'_parent', '_children'}
|
||||
and type(value)
|
||||
not in {weakref.ReferenceType, weakref.ProxyType}
|
||||
)
|
||||
}
|
||||
|
||||
# NOTE: pickling is important for parallel parsing
|
||||
def __getstate__(self):
|
||||
return self._nonrefdict()
|
||||
|
||||
def __setstate__(self, state):
|
||||
self.__dict__.update(state)
|
||||
self.children_list()
|
||||
|
||||
|
||||
ParseModel = Node
|
||||
@@ -0,0 +1,86 @@
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from .bootstrap import EBNFBootstrapParser
|
||||
from .buffering import Buffer
|
||||
from .grammars import PRAGMA_RE
|
||||
from .infos import ParserConfig
|
||||
from .parser_semantics import EBNFGrammarSemantics
|
||||
from .semantics import ASTSemantics
|
||||
|
||||
|
||||
class EBNFBuffer(Buffer):
|
||||
def __init__(
|
||||
self,
|
||||
text,
|
||||
/,
|
||||
filename=None,
|
||||
config: ParserConfig | None = None,
|
||||
**settings: Any,
|
||||
):
|
||||
config = ParserConfig.new(
|
||||
config=config, owner=self, filename=filename, **settings,
|
||||
)
|
||||
super().__init__(text, config=config)
|
||||
|
||||
def process_block(self, name, lines, index, **kwargs):
|
||||
i = 0
|
||||
while i < len(lines):
|
||||
line = lines[i]
|
||||
if re.match(PRAGMA_RE, line):
|
||||
directive, arg = line.split('#', 1)[1], ''
|
||||
if '::' in directive:
|
||||
directive, arg = directive.split('::', 1)
|
||||
directive, arg = directive.strip(), arg.strip()
|
||||
i = self.pragma(name, directive, arg, lines, index, i)
|
||||
else:
|
||||
i += 1
|
||||
return lines, index
|
||||
|
||||
def pragma(self, source, name, arg, lines, index, i):
|
||||
# we only recognize the 'include' pragama
|
||||
if name == 'include':
|
||||
filename = arg.strip('\'"')
|
||||
return self.include_file(source, filename, lines, index, i, i + 1)
|
||||
else:
|
||||
return i + 1 # will be treated as a directive by the parser
|
||||
|
||||
|
||||
class EBNFParser(EBNFBootstrapParser):
|
||||
def __init__(
|
||||
self,
|
||||
name: str | None = None,
|
||||
config: ParserConfig | None = None,
|
||||
semantics=None,
|
||||
**settings: Any,
|
||||
):
|
||||
if semantics is None:
|
||||
semantics = ASTSemantics()
|
||||
config = ParserConfig.new(
|
||||
config=config,
|
||||
name=name,
|
||||
semantics=semantics,
|
||||
tokenizercls=EBNFBuffer,
|
||||
**settings,
|
||||
)
|
||||
super().__init__(config)
|
||||
|
||||
|
||||
class GrammarGenerator(EBNFBootstrapParser):
|
||||
def __init__(
|
||||
self,
|
||||
name: str | None = None,
|
||||
config: ParserConfig | None = None,
|
||||
semantics=None,
|
||||
**settings: Any,
|
||||
):
|
||||
if semantics is None:
|
||||
semantics = EBNFGrammarSemantics(name)
|
||||
config = ParserConfig.new(
|
||||
config=config,
|
||||
name=name,
|
||||
semantics=semantics,
|
||||
tokenizercls=EBNFBuffer,
|
||||
**settings,
|
||||
)
|
||||
super().__init__(config)
|
||||
@@ -0,0 +1,148 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
from . import grammars
|
||||
from .exceptions import FailedSemantics
|
||||
from .semantics import ModelBuilderSemantics
|
||||
from .util import eval_escapes, flatten, re, warning
|
||||
|
||||
|
||||
class EBNFGrammarSemantics(ModelBuilderSemantics):
|
||||
def __init__(self, grammar_name):
|
||||
super().__init__(
|
||||
base_type=grammars.Model, types=grammars.Model.classes(),
|
||||
)
|
||||
self.grammar_name = grammar_name
|
||||
self.rules = {}
|
||||
|
||||
def token(self, ast, *args):
|
||||
token = ast
|
||||
if not token:
|
||||
raise FailedSemantics('empty token')
|
||||
return grammars.Token(token)
|
||||
|
||||
def pattern(self, ast, *args):
|
||||
return grammars.Pattern(ast)
|
||||
|
||||
def regexes(self, ast, *args):
|
||||
pattern = ''.join(ast)
|
||||
try:
|
||||
re.compile(pattern)
|
||||
except (TypeError, re.error) as e:
|
||||
raise FailedSemantics('regexp error: ' + str(e)) from e
|
||||
return ast
|
||||
|
||||
def regex(self, ast, *args):
|
||||
pattern = ast
|
||||
try:
|
||||
re.compile(pattern)
|
||||
except (TypeError, re.error) as e:
|
||||
raise FailedSemantics('regexp error: ' + str(e)) from e
|
||||
return pattern
|
||||
|
||||
def string(self, ast):
|
||||
return eval_escapes(ast)
|
||||
|
||||
def hex(self, ast):
|
||||
return int(ast, 16)
|
||||
|
||||
def float(self, ast):
|
||||
return float(ast)
|
||||
|
||||
def int(self, ast):
|
||||
return int(ast)
|
||||
|
||||
def null(self, ast):
|
||||
return None
|
||||
|
||||
def cut_deprecated(self, ast, *args):
|
||||
warning(
|
||||
'The use of >> for cut is deprecated. Use the ~ symbol instead.',
|
||||
)
|
||||
return grammars.Cut()
|
||||
|
||||
def override_single_deprecated(self, ast, *args):
|
||||
warning('The use of @ for override is deprecated. Use @: instead')
|
||||
return grammars.Override(ast)
|
||||
|
||||
def sequence(self, ast, *args):
|
||||
seq = ast.sequence
|
||||
assert isinstance(seq, Sequence), str(seq)
|
||||
if len(seq) == 1:
|
||||
return seq[0]
|
||||
return grammars.Sequence(ast)
|
||||
|
||||
def choice(self, ast, *args):
|
||||
if len(ast) == 1:
|
||||
return ast[0]
|
||||
return grammars.Choice(ast)
|
||||
|
||||
def new_name(self, name):
|
||||
if name in self.rules:
|
||||
raise FailedSemantics(f'rule "{name!s}" already defined')
|
||||
return name
|
||||
|
||||
def known_name(self, name):
|
||||
if name not in self.rules:
|
||||
raise FailedSemantics(f'rule "{name!s}" not yet defined')
|
||||
return name
|
||||
|
||||
def boolean(self, ast):
|
||||
return str(ast).lower() in {'true', 'yes', 'ok', '1'}
|
||||
|
||||
def rule(self, ast, *args):
|
||||
decorators = ast.decorators
|
||||
name = ast.name
|
||||
exp = ast.exp
|
||||
base = ast.base
|
||||
params = ast.params
|
||||
kwparams = dict(ast.kwparams) if ast.kwparams else None
|
||||
|
||||
if 'override' not in decorators and name in self.rules:
|
||||
self.new_name(name)
|
||||
elif 'override' in decorators:
|
||||
self.known_name(name)
|
||||
|
||||
if not base:
|
||||
rule = grammars.Rule(
|
||||
ast, name, exp, params, kwparams, decorators=decorators,
|
||||
)
|
||||
else:
|
||||
self.known_name(base)
|
||||
base_rule = self.rules[base]
|
||||
rule = grammars.BasedRule(
|
||||
ast,
|
||||
name,
|
||||
exp,
|
||||
base_rule,
|
||||
params,
|
||||
kwparams,
|
||||
decorators=decorators,
|
||||
)
|
||||
|
||||
self.rules[name] = rule
|
||||
return rule
|
||||
|
||||
def rule_include(self, ast, *args):
|
||||
name = str(ast)
|
||||
self.known_name(name)
|
||||
|
||||
rule = self.rules[name]
|
||||
return grammars.RuleInclude(rule)
|
||||
|
||||
def grammar(self, ast, *args):
|
||||
directives = {d.name: d.value for d in flatten(ast.directives)}
|
||||
keywords = list(flatten(ast.keywords)) or []
|
||||
|
||||
if directives.get('whitespace') in {'None', 'False'}:
|
||||
# NOTE: use '' because None will _not_ override defaults in configuration
|
||||
directives['whitespace'] = ''
|
||||
|
||||
name = self.grammar_name or directives.get('grammar')
|
||||
return grammars.Grammar(
|
||||
name,
|
||||
list(self.rules.values()),
|
||||
directives=directives,
|
||||
keywords=keywords,
|
||||
)
|
||||
@@ -0,0 +1,38 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
|
||||
from .contexts import ( # noqa: F401
|
||||
ParseContext,
|
||||
isname,
|
||||
leftrec,
|
||||
nomemo,
|
||||
tatsumasu,
|
||||
)
|
||||
from .exceptions import FailedRef
|
||||
|
||||
|
||||
class Parser(ParseContext):
|
||||
def _find_rule(self, name):
|
||||
rule = getattr(self, '_' + name + '_', None)
|
||||
if isinstance(rule, type(self._find_rule)):
|
||||
return rule
|
||||
rule = getattr(self, name, None)
|
||||
if isinstance(rule, type(self._find_rule)):
|
||||
return rule
|
||||
self._error(name, exclass=FailedRef)
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def rule_list(cls):
|
||||
methods = inspect.getmembers(cls, predicate=inspect.isroutine)
|
||||
result = []
|
||||
for m in methods:
|
||||
name = m[0]
|
||||
if len(name) < 3:
|
||||
continue
|
||||
if name.startswith('__') or name.endswith('__'):
|
||||
continue
|
||||
if name.startswith('_') and name.endswith('_'):
|
||||
result.append(name[1:-1])
|
||||
return result
|
||||
@@ -0,0 +1,141 @@
|
||||
"""
|
||||
The Renderer class provides the infrastructure for generating template-based
|
||||
code. It's used by the .grammars module for parser generation.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import itertools
|
||||
import string
|
||||
|
||||
from .util import indent, isiter, trim
|
||||
|
||||
|
||||
def render(item, join='', **fields):
|
||||
"""Render the given item"""
|
||||
if item is None:
|
||||
return ''
|
||||
elif isinstance(item, str):
|
||||
return item
|
||||
elif isinstance(item, Renderer):
|
||||
return item.render(join=join, **fields)
|
||||
elif isiter(item):
|
||||
return join.join(
|
||||
render(e, **fields) for e in iter(item) if e is not None
|
||||
)
|
||||
elif isinstance(item, int | float):
|
||||
return item
|
||||
else:
|
||||
return str(item)
|
||||
|
||||
|
||||
class RenderingFormatter(string.Formatter):
|
||||
def render(self, item, join='', **fields):
|
||||
return render(item, join=join, **fields)
|
||||
|
||||
def format_field(self, value, format_spec):
|
||||
if ':' not in format_spec:
|
||||
return super().format_field(self.render(value), format_spec)
|
||||
|
||||
ind, sep, fmt = format_spec.split(':')
|
||||
if sep == '\\n':
|
||||
sep = '\n'
|
||||
|
||||
if not ind:
|
||||
ind = 0
|
||||
mult = 0
|
||||
elif '*' in ind:
|
||||
ind, mult = ind.split('*')
|
||||
else:
|
||||
mult = 4
|
||||
ind = int(ind)
|
||||
mult = int(mult)
|
||||
|
||||
if not fmt:
|
||||
fmt = '%s'
|
||||
|
||||
if isiter(value):
|
||||
return indent(
|
||||
sep.join(fmt % self.render(v) for v in value), ind, mult,
|
||||
)
|
||||
else:
|
||||
return indent(fmt % self.render(value), ind, mult)
|
||||
|
||||
|
||||
class Renderer:
|
||||
"""Renders the fileds in the current object using a template
|
||||
provided statically, on the constructor, or as a parameter
|
||||
to render().
|
||||
|
||||
Fields with a leading underscore are not made available to
|
||||
the template. Additional fields may be made available by
|
||||
overriding render_fields().
|
||||
"""
|
||||
|
||||
template = '{__class__}'
|
||||
_counter = itertools.count()
|
||||
_formatter = RenderingFormatter()
|
||||
|
||||
def __init__(self, template=None):
|
||||
if template is not None:
|
||||
self.template = template
|
||||
|
||||
@classmethod
|
||||
def counter(cls):
|
||||
return next(cls._counter)
|
||||
|
||||
@classmethod
|
||||
def reset_counter(cls):
|
||||
Renderer._counter = itertools.count()
|
||||
|
||||
@property
|
||||
def formatter(self):
|
||||
return self._formatter
|
||||
|
||||
@formatter.setter
|
||||
def formatter(self, value):
|
||||
self._formatter = value
|
||||
|
||||
def rend(self, item, join='', **fields):
|
||||
"""A shortcut for self._formatter.render()"""
|
||||
return self._formatter.render(item, join=join, **fields)
|
||||
|
||||
def indent(self, item, ind=1, multiplier=4):
|
||||
return indent(self.rend(item), indent=ind, multiplier=multiplier)
|
||||
|
||||
def trim(self, item, tabwidth=4):
|
||||
return trim(self.rend(item), tabwidth=tabwidth)
|
||||
|
||||
def render_fields(self, fields):
|
||||
"""Pre-render fields before rendering the template."""
|
||||
return
|
||||
|
||||
def render(self, **fields):
|
||||
template = fields.pop('template', None)
|
||||
fields.update(__class__=self.__class__.__name__)
|
||||
fields.update(
|
||||
{k: v for k, v in vars(self).items() if not k.startswith('_')},
|
||||
)
|
||||
|
||||
override = self.render_fields(
|
||||
fields,
|
||||
) # pylint: disable=assignment-from-none
|
||||
if override is not None:
|
||||
template = override
|
||||
elif template is None:
|
||||
template = self.template
|
||||
|
||||
try:
|
||||
return self._formatter.format(trim(template), **fields)
|
||||
except KeyError as e:
|
||||
# find the missing key
|
||||
keys = (p[1] for p in self._formatter.parse(template))
|
||||
for key in keys:
|
||||
if key and key not in fields:
|
||||
raise KeyError(key, type(self)) from e
|
||||
raise
|
||||
|
||||
def __str__(self):
|
||||
return self.render()
|
||||
|
||||
def __repr__(self):
|
||||
return str(self)
|
||||
@@ -0,0 +1,95 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import builtins
|
||||
|
||||
from .exceptions import SemanticError
|
||||
from .objectmodel import BASE_CLASS_TOKEN, Node
|
||||
from .synth import synthesize
|
||||
from .util import simplify_list
|
||||
|
||||
|
||||
class ASTSemantics:
|
||||
def group(self, ast, *args):
|
||||
return simplify_list(ast)
|
||||
|
||||
def element(self, ast, *args):
|
||||
return simplify_list(ast)
|
||||
|
||||
def sequence(self, ast, *args):
|
||||
return simplify_list(ast)
|
||||
|
||||
def choice(self, ast, *args):
|
||||
if len(ast) == 1:
|
||||
return simplify_list(ast[0])
|
||||
return ast
|
||||
|
||||
|
||||
class ModelBuilderSemantics:
|
||||
"""Intended as a semantic action for parsing, a ModelBuilderSemantics creates
|
||||
nodes using the class name given as first parameter to a grammar
|
||||
rule, and synthesizes the class/type if it's not known.
|
||||
"""
|
||||
|
||||
def __init__(self, context=None, base_type=Node, types=None):
|
||||
self.ctx = context
|
||||
self.base_type = base_type
|
||||
|
||||
self.constructors = {}
|
||||
|
||||
for t in types or ():
|
||||
self._register_constructor(t)
|
||||
|
||||
def _register_constructor(self, constructor):
|
||||
self.constructors[constructor.__name__] = constructor
|
||||
return constructor
|
||||
|
||||
def _find_existing_constructor(self, typename):
|
||||
constructor = builtins
|
||||
for name in typename.split('.'):
|
||||
try:
|
||||
context = vars(constructor)
|
||||
except Exception as e:
|
||||
raise SemanticError(
|
||||
f'Could not find constructor for {typename} ({type(constructor).__name__}): {e!s}',
|
||||
) from e
|
||||
if name in context:
|
||||
constructor = context[name]
|
||||
else:
|
||||
constructor = None
|
||||
break
|
||||
return constructor
|
||||
|
||||
def _get_constructor(self, typename, base):
|
||||
typename = str(typename)
|
||||
|
||||
if typename in self.constructors:
|
||||
return self.constructors[typename]
|
||||
|
||||
constructor = self._find_existing_constructor(typename)
|
||||
if not constructor:
|
||||
constructor = synthesize(typename, base)
|
||||
|
||||
return self._register_constructor(constructor)
|
||||
|
||||
def _default(self, ast, *args, **kwargs):
|
||||
if not args:
|
||||
return ast
|
||||
|
||||
typespec = args[0].split(BASE_CLASS_TOKEN)
|
||||
typename = typespec[0]
|
||||
bases = typespec[-1:0:-1]
|
||||
|
||||
base = self.base_type
|
||||
for base_ in bases:
|
||||
base = self._get_constructor(base_, base)
|
||||
|
||||
constructor = self._get_constructor(typename, base)
|
||||
try:
|
||||
if isinstance(constructor, type) and issubclass(constructor, Node):
|
||||
return constructor(ast=ast, ctx=self.ctx, **kwargs)
|
||||
else:
|
||||
return constructor(ast, *args[1:], **kwargs)
|
||||
except Exception as e:
|
||||
raise SemanticError(
|
||||
f'Could not call constructor for {typename}: {e!s}',
|
||||
) from e
|
||||
@@ -0,0 +1,320 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import operator
|
||||
import weakref
|
||||
from collections import defaultdict
|
||||
from copy import copy
|
||||
|
||||
from .exceptions import ParseException
|
||||
from .infos import LineIndexInfo
|
||||
from .util import asjson, join_lists
|
||||
|
||||
DEFAULT_SEPARATOR = '.'
|
||||
|
||||
|
||||
def join_symtables(tables):
|
||||
def join_namespaces(base, target):
|
||||
base = copy(base)
|
||||
for symbol in target.symbols:
|
||||
name = symbol.name
|
||||
if name not in base or base.duplicates:
|
||||
base.insert(symbol)
|
||||
else:
|
||||
join_namespaces(base[name], symbol)
|
||||
return base
|
||||
|
||||
if not tables:
|
||||
return {}
|
||||
|
||||
result = copy(tables[0])
|
||||
for table in tables[1:]:
|
||||
result = join_namespaces(table, result)
|
||||
return result
|
||||
|
||||
|
||||
class SymbolTableError(ParseException):
|
||||
pass
|
||||
|
||||
|
||||
class Namespace:
|
||||
def __init__(
|
||||
self, ignorecase=False, duplicates=False, separator=DEFAULT_SEPARATOR,
|
||||
):
|
||||
super().__init__()
|
||||
self.ignorecase = ignorecase
|
||||
self.duplicates = duplicates
|
||||
self.separator = separator
|
||||
self._entries = defaultdict(list)
|
||||
|
||||
@property
|
||||
def entries(self):
|
||||
return self._entries
|
||||
|
||||
@property
|
||||
def symbols(self):
|
||||
return join_lists(self._entries.values())
|
||||
|
||||
@property
|
||||
def names(self):
|
||||
return list(self._entries.keys())
|
||||
|
||||
def all_names(self):
|
||||
result = []
|
||||
for name, symbols in self.entries.items():
|
||||
result.append(name)
|
||||
for symbol in symbols:
|
||||
result.extend(symbol.all_names())
|
||||
return result
|
||||
|
||||
def __contains__(self, name):
|
||||
if self.ignorecase:
|
||||
name = name.upper()
|
||||
return name in self.entries
|
||||
|
||||
def __getitem__(self, name):
|
||||
if self.ignorecase:
|
||||
name = name.upper()
|
||||
if self.duplicates:
|
||||
return self.entries.get(name)
|
||||
elif name in self.entries:
|
||||
return self.entries[name][0]
|
||||
else:
|
||||
raise KeyError(name)
|
||||
|
||||
def get(self, name, default=None):
|
||||
return self.entries.get(name, default)
|
||||
|
||||
def insert(self, symbol):
|
||||
assert isinstance(
|
||||
symbol.name, str,
|
||||
), f'"{symbol.name!s}" is not a valid symbol name'
|
||||
name = symbol.name
|
||||
if self.ignorecase:
|
||||
name = name.upper()
|
||||
if name in self._entries and not self.duplicates:
|
||||
raise SymbolTableError(
|
||||
f'Symbol "{symbol.name!s}" already in namespace',
|
||||
)
|
||||
|
||||
self._entries[name].append(symbol)
|
||||
|
||||
def lookup_all(self, qualname, drill=True):
|
||||
if self.ignorecase:
|
||||
qualname = qualname.upper()
|
||||
return self._lookup_drilldown(
|
||||
qualname.split(self.separator), drill=drill,
|
||||
)
|
||||
|
||||
def lookup(self, qualname, drill=True):
|
||||
if self.ignorecase:
|
||||
qualname = qualname.upper()
|
||||
result = self._lookup_drilldown(
|
||||
qualname.split(self.separator), drill=drill, max=1,
|
||||
)
|
||||
return result[0] if result else None
|
||||
|
||||
def _lookup_drilldown(self, namelist, drill=True, max=None):
|
||||
if not namelist:
|
||||
return []
|
||||
|
||||
result = []
|
||||
for symbol in self.symbols:
|
||||
found = symbol._lookup_drilldown(namelist, drill=drill, max=max)
|
||||
result.extend(found)
|
||||
if max and len(result) >= max:
|
||||
break
|
||||
return result
|
||||
|
||||
def resolve(self, qualname):
|
||||
return self.lookup(qualname)
|
||||
|
||||
def filter(self, condition):
|
||||
return functools.reduce(
|
||||
operator.iadd,
|
||||
(symbol.filter(condition) for symbol in self.symbols),
|
||||
[],
|
||||
)
|
||||
|
||||
def filter_first(self, condition):
|
||||
for symbol in self.symbols:
|
||||
result = symbol.filter_first(condition)
|
||||
if result:
|
||||
return result
|
||||
return None
|
||||
|
||||
def all(self):
|
||||
return self.filter(lambda: True)
|
||||
|
||||
def asjson(self):
|
||||
return asjson(self)
|
||||
|
||||
def __repr__(self):
|
||||
return f'{type(self).__name__}[]'
|
||||
|
||||
def __json__(self, seen=None):
|
||||
return {
|
||||
name: asjson(symbols, seen=seen)
|
||||
for name, symbols in self.entries.items()
|
||||
}
|
||||
|
||||
|
||||
class SymbolTable(Namespace):
|
||||
def add_reference(self, qualname, from_node):
|
||||
symbol = self.lookup(qualname)
|
||||
symbol.add_reference(qualname, from_node)
|
||||
|
||||
|
||||
class Symbol(Namespace):
|
||||
def __init__(self, name, node, ignorecase=False, duplicates=False):
|
||||
super().__init__(ignorecase=ignorecase, duplicates=duplicates)
|
||||
if not isinstance(name, str):
|
||||
raise TypeError(f'"{name}" is not a valid symbol name')
|
||||
self.name = name
|
||||
self._node = node
|
||||
self._parent = None
|
||||
self._references = []
|
||||
|
||||
@property
|
||||
def node(self):
|
||||
return self._node
|
||||
|
||||
@property
|
||||
def line(self):
|
||||
return self.node.line
|
||||
|
||||
@property
|
||||
def endline(self):
|
||||
return self.node.endline
|
||||
|
||||
@property
|
||||
def parent(self):
|
||||
if self._parent is not None:
|
||||
return self._parent()
|
||||
return None
|
||||
|
||||
@property
|
||||
def references(self):
|
||||
return self._references
|
||||
|
||||
def insert(self, symbol):
|
||||
super().insert(symbol)
|
||||
symbol._parent = weakref.ref(self)
|
||||
|
||||
def qualpath(self):
|
||||
if self.parent:
|
||||
return [*self.parent.qualpath(), self.name]
|
||||
return [self.name]
|
||||
|
||||
def qualname(self, sep=DEFAULT_SEPARATOR):
|
||||
return sep.join(self.qualpath())
|
||||
|
||||
def _lookup_drilldown(self, namelist, drill=True, max=None):
|
||||
if self.ignorecase and [self.name.upper()] == namelist:
|
||||
return [self]
|
||||
elif not self.ignorecase and [self.name] == namelist:
|
||||
return [self]
|
||||
elif self.name == namelist[0]:
|
||||
return super()._lookup_drilldown(
|
||||
namelist[1:], drill=drill, max=max,
|
||||
)
|
||||
elif drill:
|
||||
return super()._lookup_drilldown(namelist, drill=drill, max=max)
|
||||
else:
|
||||
return []
|
||||
|
||||
def resolve(self, qualname):
|
||||
return (
|
||||
self.lookup(qualname) or
|
||||
(self.parent and self.parent.resolve(qualname))
|
||||
)
|
||||
|
||||
def filter(self, condition):
|
||||
this_case = [self] if condition(self) else []
|
||||
return this_case + super().filter(condition)
|
||||
|
||||
def filter_first(self, condition):
|
||||
if condition(self):
|
||||
return self
|
||||
return super().filter(condition)
|
||||
|
||||
def add_reference(self, qualname, node):
|
||||
# reference = SymbolReference(self, qualname, node)
|
||||
reference = node
|
||||
if reference not in self.references:
|
||||
self._references.append(reference)
|
||||
|
||||
def line_index(self, include_entries=False, include_references=False):
|
||||
result = set(self.node.line_index())
|
||||
|
||||
if include_references:
|
||||
result.update(self.reference_line_index())
|
||||
|
||||
if include_entries:
|
||||
for s in self.symbols:
|
||||
index = s.line_index(
|
||||
include_entries=include_entries,
|
||||
include_references=include_references,
|
||||
)
|
||||
result.update(index)
|
||||
assert isinstance(result, set)
|
||||
assert all(isinstance(i, LineIndexInfo) for i in result)
|
||||
return sorted(result)
|
||||
|
||||
def reference_line_index(self):
|
||||
result = set()
|
||||
for r in self.references:
|
||||
result.update(r.line_index())
|
||||
assert isinstance(result, set)
|
||||
assert all(isinstance(i, LineIndexInfo) for i in result)
|
||||
return result
|
||||
|
||||
def __repr__(self):
|
||||
return f'{self.name}[]'
|
||||
|
||||
def __json__(self, seen=None):
|
||||
return dict(
|
||||
('node', type(self.node).__name__),
|
||||
('entries', super().__json__(seen=seen)),
|
||||
('references', asjson(self._references, seen=seen)),
|
||||
)
|
||||
|
||||
def __getstate__(self):
|
||||
state = self.__dict__.copy()
|
||||
state.update(_parent=self.parent)
|
||||
return state
|
||||
|
||||
def __setstate__(self, state):
|
||||
self.__dict__.update(state)
|
||||
if self._parent is not None:
|
||||
self._parent = weakref.ref(self._parent)
|
||||
|
||||
|
||||
class BasedSymbol(Namespace):
|
||||
def __init__(self, name, node, duplicates=False):
|
||||
super().__init__(duplicates=duplicates)
|
||||
self._bases = []
|
||||
|
||||
@property
|
||||
def bases(self):
|
||||
return self._bases
|
||||
|
||||
def add_base(self, base):
|
||||
assert isinstance(base, Symbol)
|
||||
self._bases.append(base)
|
||||
|
||||
def _lookup_drilldown(self, namelist, drill=True, max=max):
|
||||
result = super()._lookup_drilldown(namelist, drill=drill, max=max)
|
||||
if result:
|
||||
return result
|
||||
|
||||
for base in self.bases:
|
||||
result = base._lookup_drilldown(namelist, drill=drill, max=max)
|
||||
if result:
|
||||
return result
|
||||
return result
|
||||
|
||||
def __json__(self, seen=None):
|
||||
result = super().__json__(seen=seen)
|
||||
result['bases'] = asjson([b.qualname() for b in self.bases], seen=seen)
|
||||
return result
|
||||
@@ -0,0 +1,31 @@
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
__REGISTRY: Mapping[str, Any] = vars()
|
||||
|
||||
|
||||
class _Synthetic:
|
||||
def __reduce__(self):
|
||||
return (
|
||||
synthesize(type(self).__name__, type(self).__bases__),
|
||||
(),
|
||||
vars(self),
|
||||
)
|
||||
|
||||
|
||||
def synthesize(name, bases):
|
||||
typename = f'{__name__}.{name}'
|
||||
|
||||
if not isinstance(bases, tuple):
|
||||
bases = (bases,)
|
||||
|
||||
if _Synthetic not in bases:
|
||||
bases += (_Synthetic,)
|
||||
|
||||
constructor = __REGISTRY.get(typename)
|
||||
if not constructor:
|
||||
constructor = type(name, bases, {})
|
||||
typename = constructor.__name__
|
||||
__REGISTRY[typename] = constructor
|
||||
|
||||
return constructor
|
||||
@@ -0,0 +1,69 @@
|
||||
from .exceptions import ParseError
|
||||
from .util._common import _prints
|
||||
|
||||
|
||||
class Tokenizer:
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__()
|
||||
|
||||
def error(self, *args, **kwargs):
|
||||
raise ParseError(_prints(*args, **kwargs))
|
||||
|
||||
@property
|
||||
def filename(self):
|
||||
raise NotImplementedError
|
||||
|
||||
@property
|
||||
def ignorecase(self):
|
||||
raise NotImplementedError
|
||||
|
||||
@property
|
||||
def pos(self):
|
||||
raise NotImplementedError
|
||||
|
||||
def goto(self, pos):
|
||||
raise NotImplementedError
|
||||
|
||||
def atend(self):
|
||||
raise NotImplementedError
|
||||
|
||||
def ateol(self):
|
||||
raise NotImplementedError
|
||||
|
||||
@property
|
||||
def current(self):
|
||||
raise NotImplementedError
|
||||
|
||||
@property
|
||||
def token(self):
|
||||
return self.current
|
||||
|
||||
def next(self):
|
||||
raise NotImplementedError
|
||||
|
||||
def next_token(self):
|
||||
raise NotImplementedError
|
||||
|
||||
def match(self, token):
|
||||
raise NotImplementedError
|
||||
|
||||
def matchre(self, pattern):
|
||||
raise NotImplementedError
|
||||
|
||||
def posline(self, pos):
|
||||
raise NotImplementedError
|
||||
|
||||
def line_info(self, pos=None):
|
||||
raise NotImplementedError
|
||||
|
||||
def get_lines(self, start=None, end=None):
|
||||
raise NotImplementedError
|
||||
|
||||
def lookahead(self):
|
||||
raise NotImplementedError
|
||||
|
||||
def lookahead_pos(self):
|
||||
if self.atend():
|
||||
return ''
|
||||
info = self.line_info()
|
||||
return '~%d:%d' % (info.line + 1, info.col + 1)
|
||||
@@ -0,0 +1,356 @@
|
||||
"""
|
||||
Parse and translate an EBNF grammar into a Python parser for
|
||||
the described language.
|
||||
"""
|
||||
# ruff: noqa: PLR0912
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import codecs
|
||||
import importlib
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from ._version import __version__
|
||||
from .exceptions import ParseException
|
||||
from .infos import ParserConfig
|
||||
from .ngcodegen import codegen as ngpythoncg
|
||||
from .ngcodegen import objectmodel as ngobjectmodel
|
||||
from .parser import GrammarGenerator
|
||||
from .semantics import ModelBuilderSemantics
|
||||
from .util import eval_escapes
|
||||
|
||||
DESCRIPTION = (
|
||||
'TatSu takes a grammar'
|
||||
' in a variation of EBNF as input, and outputs a memoizing'
|
||||
' PEG/Packrat parser in Python.'
|
||||
)
|
||||
|
||||
|
||||
def parse_args():
|
||||
argparser = argparse.ArgumentParser(
|
||||
prog='tatsu', description=DESCRIPTION, add_help=False,
|
||||
)
|
||||
|
||||
main_mode = argparser.add_mutually_exclusive_group()
|
||||
main_mode.add_argument(
|
||||
'--generate-parser',
|
||||
help='generate parser code from the grammar (default)',
|
||||
action='store_true',
|
||||
)
|
||||
main_mode.add_argument(
|
||||
'--draw',
|
||||
'-d',
|
||||
help='generate a diagram of the grammar (requires --outfile)',
|
||||
action='store_true',
|
||||
)
|
||||
main_mode.add_argument(
|
||||
'--object-model',
|
||||
'-g',
|
||||
help='generate object model from the class names given as rule arguments',
|
||||
action='store_true',
|
||||
)
|
||||
main_mode.add_argument(
|
||||
'--pretty',
|
||||
'-p',
|
||||
help='generate a prettified version of the input grammar',
|
||||
action='store_true',
|
||||
)
|
||||
main_mode.add_argument(
|
||||
'--pretty-lean',
|
||||
help='like --pretty, but without name: or ::Parameter annotations',
|
||||
action='store_true',
|
||||
)
|
||||
|
||||
ebnf_opts = argparser.add_argument_group('parse-time options')
|
||||
argparser.add_argument(
|
||||
'filename',
|
||||
metavar='GRAMMAR',
|
||||
help='the filename of the TatSu grammar to parse',
|
||||
)
|
||||
ebnf_opts.add_argument(
|
||||
'--color',
|
||||
'-c',
|
||||
help='use color in traces (requires the colorama library)',
|
||||
action='store_true',
|
||||
)
|
||||
ebnf_opts.add_argument(
|
||||
'--trace',
|
||||
'-t',
|
||||
help='produce verbose parsing output',
|
||||
action='store_true',
|
||||
)
|
||||
|
||||
generation_opts = argparser.add_argument_group('generation options')
|
||||
generation_opts.add_argument(
|
||||
'--no-left-recursion',
|
||||
'-l',
|
||||
help='turns left-recursion support off',
|
||||
dest='left_recursion',
|
||||
action='store_false',
|
||||
default=True,
|
||||
)
|
||||
generation_opts.add_argument(
|
||||
'--name',
|
||||
'-m',
|
||||
metavar='NAME',
|
||||
help='Name for the grammar (defaults to GRAMMAR base name)',
|
||||
)
|
||||
generation_opts.add_argument(
|
||||
'--no-nameguard',
|
||||
'-n',
|
||||
help='allow tokens that are prefixes of others',
|
||||
dest='nameguard',
|
||||
action='store_false',
|
||||
default=None, # None allows grammar specified
|
||||
)
|
||||
generation_opts.add_argument(
|
||||
'--outfile',
|
||||
'--output',
|
||||
'-o',
|
||||
metavar='FILE',
|
||||
help='output file (default is stdout)',
|
||||
)
|
||||
generation_opts.add_argument(
|
||||
'--object-model-outfile',
|
||||
'-G',
|
||||
metavar='FILE',
|
||||
help='generate object model and save to FILE',
|
||||
)
|
||||
generation_opts.add_argument(
|
||||
'--whitespace',
|
||||
'-w',
|
||||
metavar='CHARACTERS',
|
||||
help='characters to skip during parsing (use "" to disable)',
|
||||
)
|
||||
|
||||
def import_class(path):
|
||||
try:
|
||||
spath = path.rsplit('.', 1)
|
||||
module = importlib.import_module(spath[0])
|
||||
|
||||
return getattr(module, spath[1])
|
||||
except Exception as e:
|
||||
raise argparse.ArgumentTypeError(
|
||||
f"Couldn't find class {path}",
|
||||
) from e
|
||||
|
||||
generation_opts.add_argument(
|
||||
'--base-type',
|
||||
metavar='CLASSPATH',
|
||||
help='class to use as base type for the object model, for example "mymodule.MyNode"',
|
||||
type=import_class,
|
||||
)
|
||||
|
||||
std_args = argparser.add_argument_group('common options')
|
||||
std_args.add_argument(
|
||||
'--help', '-h', help='show this help message and exit', action='help',
|
||||
)
|
||||
std_args.add_argument(
|
||||
'--version',
|
||||
'-V',
|
||||
help='provide version information and exit',
|
||||
action='version',
|
||||
version=__version__,
|
||||
)
|
||||
|
||||
args = argparser.parse_args()
|
||||
|
||||
if args.draw and not args.outfile:
|
||||
argparser.error('--draw requires --outfile')
|
||||
|
||||
return args
|
||||
|
||||
|
||||
__compiled_grammar_cache = {} # type: ignore[var-annotated]
|
||||
|
||||
|
||||
def compile(
|
||||
grammar,
|
||||
name=None,
|
||||
semantics=None,
|
||||
asmodel=False,
|
||||
config: ParserConfig | None = None,
|
||||
**settings,
|
||||
):
|
||||
cache = __compiled_grammar_cache
|
||||
|
||||
key = (name, grammar, id(semantics))
|
||||
if key in cache:
|
||||
model = cache[key]
|
||||
else:
|
||||
gen = GrammarGenerator(name, config=config, **settings)
|
||||
model = cache[key] = gen.parse(grammar, config=config, **settings)
|
||||
|
||||
if semantics is not None:
|
||||
model.semantics = semantics
|
||||
elif asmodel:
|
||||
model.semantics = ModelBuilderSemantics()
|
||||
|
||||
return model
|
||||
|
||||
|
||||
def parse(
|
||||
grammar,
|
||||
input,
|
||||
start=None,
|
||||
name=None,
|
||||
semantics=None,
|
||||
asmodel=False,
|
||||
config: ParserConfig | None = None,
|
||||
**settings,
|
||||
):
|
||||
model = compile(
|
||||
grammar,
|
||||
name=name,
|
||||
semantics=semantics,
|
||||
asmodel=asmodel,
|
||||
config=config,
|
||||
**settings,
|
||||
)
|
||||
return model.parse(
|
||||
input, start=start, semantics=semantics, config=config, **settings,
|
||||
)
|
||||
|
||||
|
||||
def to_python_sourcecode(
|
||||
grammar,
|
||||
name=None,
|
||||
filename=None,
|
||||
config: ParserConfig | None = None,
|
||||
**settings,
|
||||
):
|
||||
model = compile(
|
||||
grammar, name=name, filename=filename, config=config, **settings,
|
||||
)
|
||||
return ngpythoncg(model)
|
||||
|
||||
|
||||
def to_python_model(
|
||||
grammar,
|
||||
name=None,
|
||||
filename=None,
|
||||
base_type=None,
|
||||
config: ParserConfig | None = None,
|
||||
**settings,
|
||||
):
|
||||
model = compile(
|
||||
grammar, name=name, filename=filename, config=config, **settings,
|
||||
)
|
||||
return ngobjectmodel.modelgen(model, base_type=base_type)
|
||||
|
||||
|
||||
# for backwards compatibility. Use `compile()` instead
|
||||
def genmodel(
|
||||
name=None,
|
||||
grammar=None,
|
||||
semantics=None,
|
||||
config: ParserConfig | None = None,
|
||||
**settings,
|
||||
):
|
||||
if grammar is None:
|
||||
raise ParseException('grammar is None')
|
||||
|
||||
return compile(
|
||||
grammar, name=name, semantics=semantics, config=config, **settings,
|
||||
)
|
||||
|
||||
|
||||
def gencode(
|
||||
name=None,
|
||||
grammar=None,
|
||||
trace=False,
|
||||
filename=None,
|
||||
codegen=ngpythoncg,
|
||||
config: ParserConfig | None = None,
|
||||
**settings,
|
||||
):
|
||||
model = compile(
|
||||
grammar,
|
||||
name=name,
|
||||
filename=filename,
|
||||
trace=trace,
|
||||
config=config,
|
||||
**settings,
|
||||
)
|
||||
return codegen(model)
|
||||
|
||||
|
||||
def prepare_for_output(filename):
|
||||
if filename:
|
||||
filename = Path(filename)
|
||||
if filename.is_file():
|
||||
filename.unlink()
|
||||
dirname = filename.parent
|
||||
if dirname.exists():
|
||||
dirname.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
def save(filename, content):
|
||||
with codecs.open(filename, 'w', encoding='utf-8') as f:
|
||||
f.write(content)
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
|
||||
if args.whitespace:
|
||||
args.whitespace = eval_escapes(args.whitespace)
|
||||
|
||||
outfile = args.outfile
|
||||
prepare_for_output(outfile)
|
||||
prepare_for_output(args.object_model_outfile)
|
||||
|
||||
grammar = codecs.open(args.filename, encoding='utf-8').read()
|
||||
|
||||
try:
|
||||
model = compile(
|
||||
grammar,
|
||||
args.name,
|
||||
trace=args.trace,
|
||||
filename=args.filename,
|
||||
colorize=args.color,
|
||||
)
|
||||
model.whitespace = args.whitespace
|
||||
model.nameguard = args.nameguard
|
||||
model.left_recursion = args.left_recursion
|
||||
|
||||
if args.draw:
|
||||
from tatsu import diagrams
|
||||
|
||||
diagrams.draw(outfile, model)
|
||||
else:
|
||||
if args.pretty:
|
||||
result = model.pretty()
|
||||
elif args.pretty_lean:
|
||||
result = model.pretty_lean()
|
||||
elif args.object_model:
|
||||
result = ngobjectmodel.modelgen(model, base_type=args.base_type)
|
||||
else:
|
||||
result = ngpythoncg(model)
|
||||
|
||||
if outfile:
|
||||
save(outfile, result)
|
||||
else:
|
||||
print(result)
|
||||
|
||||
# if requested, always save it
|
||||
if args.object_model_outfile:
|
||||
save(
|
||||
args.object_model_outfile,
|
||||
ngobjectmodel.modelgen(model, base_type=args.base_type),
|
||||
)
|
||||
|
||||
print('-' * 72, file=sys.stderr)
|
||||
print(
|
||||
f'{len(grammar.split()):12,d} lines in grammar', file=sys.stderr,
|
||||
)
|
||||
print(f'{len(model.rules):12,d} rules in grammar', file=sys.stderr)
|
||||
print(f'{model.nodecount():12,d} nodes in AST', file=sys.stderr)
|
||||
except ParseException as e:
|
||||
print(e, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1 @@
|
||||
from ._common import * # noqa: F403
|
||||
@@ -0,0 +1,540 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import codecs
|
||||
import datetime
|
||||
import enum
|
||||
import functools
|
||||
import json
|
||||
import keyword
|
||||
import logging
|
||||
import operator
|
||||
import os
|
||||
import os.path
|
||||
import re
|
||||
import sys
|
||||
import warnings
|
||||
import weakref
|
||||
from collections.abc import Iterable, Mapping, MutableSequence
|
||||
from io import StringIO
|
||||
from itertools import zip_longest
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger('tatsu')
|
||||
logger.setLevel(logging.DEBUG)
|
||||
ch = logging.StreamHandler()
|
||||
formatter = logging.Formatter('%(message)s')
|
||||
ch.setFormatter(formatter)
|
||||
logger.addHandler(ch)
|
||||
|
||||
|
||||
ESCAPE_SEQUENCE_RE = re.compile(
|
||||
r"""
|
||||
( \\U........ # 8-digit Unicode escapes
|
||||
| \\u.... # 4-digit Unicode escapes
|
||||
| \\x.. # 2-digit Unicode escapes
|
||||
| \\[0-7]{1,3} # Octal character escapes
|
||||
| \\N\{[^}]+\} # Unicode characters by name
|
||||
| \\[\\'"abfnrtv] # Single-character escapes
|
||||
)""",
|
||||
re.UNICODE | re.VERBOSE,
|
||||
)
|
||||
|
||||
|
||||
class AsJSONMixin:
|
||||
def __json__(self, seen=None):
|
||||
return {
|
||||
'__class__': type(self).__name__,
|
||||
**asjson(self._pubdict(), seen=seen),
|
||||
}
|
||||
|
||||
def _pubdict(self):
|
||||
return {
|
||||
name: value
|
||||
for name, value in vars(self).items()
|
||||
if not name.startswith('_')
|
||||
}
|
||||
|
||||
|
||||
def program_name():
|
||||
import __main__ as main
|
||||
if package := main.__package__:
|
||||
return package
|
||||
return Path(main.__file__).name
|
||||
|
||||
|
||||
def is_posix():
|
||||
return os.name == 'posix'
|
||||
|
||||
|
||||
def _prints(*args, **kwargs):
|
||||
with StringIO() as f:
|
||||
kwargs['file'] = f
|
||||
kwargs['end'] = ''
|
||||
print(*args, **kwargs)
|
||||
return f.getvalue()
|
||||
|
||||
|
||||
def info(*args, **kwargs):
|
||||
logger.info(_prints(*args, **kwargs))
|
||||
|
||||
|
||||
def debug(*args, **kwargs):
|
||||
logger.debug(_prints(*args, **kwargs))
|
||||
|
||||
|
||||
def warning(*args, **kwargs):
|
||||
logger.warning(_prints(*args, **kwargs))
|
||||
|
||||
|
||||
def error(*args, **kwargs):
|
||||
logger.error(_prints(*args, **kwargs))
|
||||
|
||||
|
||||
def identity(*args):
|
||||
if len(args) == 1:
|
||||
return args[0]
|
||||
return args
|
||||
|
||||
|
||||
def is_list(o):
|
||||
return type(o) is list
|
||||
|
||||
|
||||
def to_list(o):
|
||||
if o is None:
|
||||
return []
|
||||
elif isinstance(o, list):
|
||||
return o
|
||||
else:
|
||||
return [o]
|
||||
|
||||
|
||||
def is_namedtuple(obj) -> bool:
|
||||
return (
|
||||
len(type(obj).__bases__) == 1
|
||||
and isinstance(obj, tuple)
|
||||
and hasattr(obj, '_asdict')
|
||||
and hasattr(obj, '_fields')
|
||||
and all(isinstance(f, str) for f in getattr(obj, '_fields', ()))
|
||||
)
|
||||
|
||||
|
||||
def simplify_list(x):
|
||||
if isinstance(x, list) and len(x) == 1:
|
||||
return simplify_list(x[0])
|
||||
return x
|
||||
|
||||
|
||||
def extend_list(x, n, default=None):
|
||||
def _null():
|
||||
pass
|
||||
|
||||
default = default or _null
|
||||
|
||||
missing = max(0, 1 + n - len(x))
|
||||
x.extend(default() for _ in range(missing))
|
||||
|
||||
|
||||
def contains_sublist(lst, sublst):
|
||||
n = len(sublst)
|
||||
return any(sublst == lst[i: i + n] for i in range(1 + len(lst) - n))
|
||||
|
||||
|
||||
def join_lists(lists):
|
||||
return functools.reduce(operator.iadd, lists, [])
|
||||
|
||||
|
||||
def flatten(o):
|
||||
if not isinstance(o, MutableSequence):
|
||||
yield o
|
||||
return
|
||||
|
||||
for item in o:
|
||||
yield from flatten(item)
|
||||
|
||||
|
||||
def compress_seq(seq):
|
||||
seen = set()
|
||||
result = []
|
||||
for x in seq:
|
||||
if x not in seen:
|
||||
result.append(x)
|
||||
seen.add(x)
|
||||
return result
|
||||
|
||||
|
||||
def eval_escapes(s):
|
||||
"""
|
||||
Given a string, evaluate escape sequences starting with backslashes as
|
||||
they would be evaluated in Python source code. For a list of these
|
||||
sequences, see: https://docs.python.org/3/reference/lexical_analysis.html
|
||||
|
||||
This is not the same as decoding the whole string with the 'unicode-escape'
|
||||
codec, because that provides no way to handle non-ASCII characters that are
|
||||
literally present in the string.
|
||||
"""
|
||||
# by Rob Speer
|
||||
|
||||
def decode_match(match):
|
||||
return codecs.decode(match.group(0), 'unicode-escape')
|
||||
|
||||
return ESCAPE_SEQUENCE_RE.sub(decode_match, s)
|
||||
|
||||
|
||||
def isiter(value):
|
||||
return isinstance(value, Iterable) and not isinstance(
|
||||
value, str | bytes | bytearray,
|
||||
)
|
||||
|
||||
|
||||
def trim(text, tabwidth=4):
|
||||
"""
|
||||
Trim text of common, leading whitespace.
|
||||
|
||||
Based on the trim algorithm of PEP 257:
|
||||
http://www.python.org/dev/peps/pep-0257/
|
||||
"""
|
||||
if not text:
|
||||
return ''
|
||||
lines = text.expandtabs(tabwidth).splitlines()
|
||||
maxindent = len(text)
|
||||
indent = maxindent
|
||||
for line in lines[1:]:
|
||||
stripped = line.lstrip()
|
||||
if stripped:
|
||||
indent = min(indent, len(line) - len(stripped))
|
||||
trimmed = [lines[0].strip()] + [
|
||||
line[indent:].rstrip() for line in lines[1:]
|
||||
]
|
||||
i = 0
|
||||
while i < len(trimmed) and not trimmed[i]:
|
||||
i += 1
|
||||
return '\n'.join(trimmed[i:])
|
||||
|
||||
|
||||
def indent(text, indent=1, multiplier=4):
|
||||
"""Indent the given block of text by indent*4 spaces"""
|
||||
if text is None:
|
||||
return ''
|
||||
text = str(text)
|
||||
if indent >= 0:
|
||||
sindent = ' ' * multiplier * indent
|
||||
text = '\n'.join((sindent + t).rstrip() for t in text.splitlines())
|
||||
return text
|
||||
|
||||
|
||||
def format_if(fmt, values):
|
||||
return fmt % values if values else ''
|
||||
|
||||
|
||||
def notnone(value, default=None):
|
||||
return value if value is not None else default
|
||||
|
||||
|
||||
def timestamp():
|
||||
return '.'.join(
|
||||
'%2.2d' % t for t in datetime.datetime.utcnow().utctimetuple()[:-2]
|
||||
)
|
||||
|
||||
|
||||
def asjson(obj, seen=None): # noqa: PLR0911, PLR0912
|
||||
if obj is None or isinstance(obj, int | float | str | bool):
|
||||
return obj
|
||||
|
||||
if seen is None:
|
||||
seen = set()
|
||||
elif id(obj) in seen:
|
||||
return f'{type(obj).__name__}@{id(obj)}'
|
||||
|
||||
if isinstance(obj, Mapping | AsJSONMixin) or isiter(obj):
|
||||
seen.add(id(obj))
|
||||
|
||||
try:
|
||||
if isinstance(obj, weakref.ReferenceType | weakref.ProxyType):
|
||||
return f'{obj.__class__.__name__}@0x{hex(id(obj)).upper()[2:]}'
|
||||
elif hasattr(obj, '__json__'):
|
||||
return obj.__json__(seen=seen)
|
||||
elif is_namedtuple(obj):
|
||||
return asjson(obj._asdict(), seen=seen)
|
||||
elif isinstance(obj, Mapping):
|
||||
result = {}
|
||||
for k, v in obj.items():
|
||||
try:
|
||||
result[k] = asjson(v, seen=seen)
|
||||
except TypeError:
|
||||
debug('Unhashable key?', type(k), str(k))
|
||||
raise
|
||||
return result
|
||||
elif isiter(obj):
|
||||
return [asjson(e, seen=seen) for e in obj]
|
||||
elif isinstance(obj, enum.Enum):
|
||||
return asjson(obj.value)
|
||||
else:
|
||||
return repr(obj)
|
||||
finally:
|
||||
# NOTE: id()s may be reused
|
||||
# https://docs.python.org/3/library/functions.html#id
|
||||
seen -= {id(obj)}
|
||||
|
||||
|
||||
def minjson(obj, typesfiltered=(str, list, dict)):
|
||||
if isinstance(obj, Mapping):
|
||||
return {
|
||||
name: minjson(value)
|
||||
for name, value in obj.items()
|
||||
if (
|
||||
not name.startswith('_')
|
||||
and value is not None
|
||||
and (value or not isinstance(value, typesfiltered))
|
||||
)
|
||||
}
|
||||
elif isiter(obj):
|
||||
return [minjson(e) for e in obj]
|
||||
else:
|
||||
return obj
|
||||
|
||||
|
||||
def plainjson(obj):
|
||||
if isinstance(obj, Mapping):
|
||||
return {
|
||||
name: plainjson(value)
|
||||
for name, value in obj.items()
|
||||
if name not in {'__class__', 'parseinfo'}
|
||||
}
|
||||
elif isinstance(obj, weakref.ReferenceType | weakref.ProxyType):
|
||||
return '@ref'
|
||||
elif isinstance(obj, str) and obj.startswith('@'):
|
||||
return '@ref'
|
||||
elif isiter(obj):
|
||||
return [plainjson(e) for e in obj]
|
||||
else:
|
||||
return obj
|
||||
|
||||
|
||||
class FallbackJSONEncoder(json.JSONEncoder):
|
||||
"""A JSON Encoder that falls back to repr() for non-JSON-aware objects."""
|
||||
|
||||
def default(self, o):
|
||||
return repr(o)
|
||||
|
||||
|
||||
def asjsons(obj):
|
||||
return json.dumps(asjson(obj), indent=2, cls=FallbackJSONEncoder)
|
||||
|
||||
|
||||
def prune_dict(d, predicate):
|
||||
"""Remove all items x where predicate(x, d[x])"""
|
||||
|
||||
keys = [k for k, v in d.items() if predicate(k, v)]
|
||||
for k in keys:
|
||||
del d[k]
|
||||
|
||||
|
||||
def safe_name(name):
|
||||
if keyword.iskeyword(name):
|
||||
return name + '_'
|
||||
return name
|
||||
|
||||
|
||||
def chunks(iterable, size, fillvalue=None):
|
||||
return zip_longest(*[iter(iterable)] * size, fillvalue=fillvalue)
|
||||
|
||||
|
||||
def generic_main(custom_main, parser_class, name='Unknown'):
|
||||
import argparse
|
||||
|
||||
class ListRules(argparse.Action):
|
||||
def __call__(self, parser, namespace, values, option_string=None):
|
||||
print('Rules:')
|
||||
for r in parser_class.rule_list():
|
||||
print(r)
|
||||
print()
|
||||
sys.exit(0)
|
||||
|
||||
argp = argparse.ArgumentParser(description=f'Simple parser for {name}.')
|
||||
addarg = argp.add_argument
|
||||
|
||||
addarg(
|
||||
'-c',
|
||||
'--color',
|
||||
help='use color in traces (requires the colorama library)',
|
||||
action='store_true',
|
||||
)
|
||||
addarg(
|
||||
'-l',
|
||||
'--list',
|
||||
action=ListRules,
|
||||
nargs=0,
|
||||
help='list all rules and exit',
|
||||
)
|
||||
addarg(
|
||||
'-n',
|
||||
'--no-nameguard',
|
||||
action='store_true',
|
||||
dest='no_nameguard',
|
||||
help="disable the 'nameguard' feature",
|
||||
)
|
||||
addarg(
|
||||
'-t', '--trace', action='store_true', help='output trace information',
|
||||
)
|
||||
addarg(
|
||||
'-w',
|
||||
'--whitespace',
|
||||
type=str,
|
||||
default=None,
|
||||
help='whitespace specification',
|
||||
)
|
||||
addarg(
|
||||
'file',
|
||||
metavar='FILE',
|
||||
help="the input file to parse or '-' for standard input",
|
||||
nargs='?',
|
||||
default='-',
|
||||
)
|
||||
addarg(
|
||||
'startrule',
|
||||
metavar='STARTRULE',
|
||||
nargs='?',
|
||||
help='the start rule for parsing',
|
||||
default=None,
|
||||
)
|
||||
|
||||
args = argp.parse_args()
|
||||
try:
|
||||
return custom_main(
|
||||
args.file,
|
||||
start=args.startrule,
|
||||
trace=args.trace,
|
||||
whitespace=args.whitespace,
|
||||
nameguard=not args.no_nameguard,
|
||||
colorize=args.color,
|
||||
)
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
|
||||
|
||||
# decorator
|
||||
def deprecated(fun):
|
||||
@functools.wraps(fun)
|
||||
def wrapper(*args, **kwargs):
|
||||
warnings.filterwarnings('default', category=DeprecationWarning)
|
||||
msg = 'Call to deprecated function {}.'
|
||||
warnings.warn(
|
||||
msg.format(fun.__name__), category=DeprecationWarning, stacklevel=2,
|
||||
)
|
||||
return fun(*args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
def left_assoc(elements):
|
||||
if not elements:
|
||||
return ()
|
||||
|
||||
it = iter(elements)
|
||||
expre = next(it)
|
||||
for e in it:
|
||||
op = e
|
||||
expre = (op, expre, next(it))
|
||||
return expre
|
||||
|
||||
|
||||
def right_assoc(elements):
|
||||
if not elements:
|
||||
return ()
|
||||
|
||||
def assoc(it):
|
||||
left = next(it)
|
||||
try:
|
||||
op = next(it)
|
||||
except StopIteration:
|
||||
return left
|
||||
else:
|
||||
return (op, left, assoc(it))
|
||||
|
||||
return assoc(iter(elements))
|
||||
|
||||
|
||||
try:
|
||||
import psutil
|
||||
except ImportError:
|
||||
|
||||
def memory_use():
|
||||
return 0
|
||||
|
||||
else:
|
||||
|
||||
def memory_use():
|
||||
process = psutil.Process(os.getpid())
|
||||
return process.memory_info().rss
|
||||
|
||||
|
||||
def try_read(filename):
|
||||
if isinstance(filename, Path):
|
||||
filename = str(filename)
|
||||
for e in ['utf-16', 'utf-8', 'latin-1', 'cp1252', 'ascii']:
|
||||
try:
|
||||
with Path(filename).open(encoding=e) as f:
|
||||
return f.read()
|
||||
except UnicodeError:
|
||||
pass
|
||||
raise UnicodeDecodeError(f"cannot find the encoding for '{filename}'")
|
||||
|
||||
|
||||
def filelist_from_patterns(patterns, ignore=None, base='.', sizesort=False):
|
||||
base = Path(base or '.').expanduser()
|
||||
|
||||
filenames = set()
|
||||
for pattern in patterns or []:
|
||||
path = base / pattern
|
||||
if path.is_file():
|
||||
filenames.add(path)
|
||||
continue
|
||||
|
||||
if path.is_dir():
|
||||
path += '/*'
|
||||
|
||||
parts = path.parts[1:] if path.is_absolute() else path.parts
|
||||
joined_pattern = str(Path().joinpath(*parts))
|
||||
filenames.update(
|
||||
p for p in Path(path.root).glob(joined_pattern) if not p.is_dir()
|
||||
)
|
||||
|
||||
filenames = list(filenames)
|
||||
|
||||
def excluded(path):
|
||||
if any(path.match(ex) for ex in ignore):
|
||||
return True
|
||||
|
||||
return any(
|
||||
any(Path(part).match(ex) for ex in ignore) for part in path.parts
|
||||
)
|
||||
|
||||
if ignore:
|
||||
filenames = [path for path in filenames if not excluded(path)]
|
||||
if sizesort:
|
||||
filenames.sort(key=lambda f: f.stat().st_size)
|
||||
|
||||
return filenames
|
||||
|
||||
|
||||
def short_relative_path(path, base='.'):
|
||||
path = Path(path)
|
||||
base = Path(base)
|
||||
common = Path(os.path.commonpath([base.resolve(), path.resolve()]))
|
||||
|
||||
if common == path.root:
|
||||
return path
|
||||
elif common == Path.home():
|
||||
up = Path('~')
|
||||
elif common == base:
|
||||
up = Path()
|
||||
else:
|
||||
n = len(base.parts) - len(common.parts)
|
||||
up = Path('../' * n)
|
||||
|
||||
rel = str(up / path.resolve().relative_to(common))
|
||||
if len(rel) < len(str(path)):
|
||||
return rel
|
||||
else:
|
||||
return str(path)
|
||||
@@ -0,0 +1,117 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Iterable
|
||||
from functools import cache
|
||||
from typing import TypeVar
|
||||
|
||||
_T = TypeVar('_T')
|
||||
|
||||
_undefined = object() # unique object for when None is not a good default
|
||||
|
||||
|
||||
def first(iterable, default=_undefined):
|
||||
"""Return the first item of *iterable*, or *default* if *iterable* is
|
||||
empty.
|
||||
|
||||
>>> first([0, 1, 2, 3])
|
||||
0
|
||||
>>> first([], 'some default')
|
||||
'some default'
|
||||
|
||||
If *default* is not provided and there are no items in the iterable,
|
||||
raise ``ValueError``.
|
||||
|
||||
:func:`first` is useful when you have a generator of expensive-to-retrieve
|
||||
values and want any arbitrary one. It is marginally shorter than
|
||||
``next(iter(iterable), default)``.
|
||||
|
||||
"""
|
||||
# NOTE: https://more-itertools.readthedocs.io/en/stable/_modules/more_itertools/more.html#first
|
||||
try:
|
||||
return next(iter(iterable))
|
||||
except StopIteration as e:
|
||||
# I'm on the edge about raising ValueError instead of StopIteration. At
|
||||
# the moment, ValueError wins, because the caller could conceivably
|
||||
# want to do something different with flow control when I raise the
|
||||
# exception, and it's weird to explicitly catch StopIteration.
|
||||
if default is _undefined:
|
||||
raise ValueError(
|
||||
'first() was called on an empty iterable, and no '
|
||||
'default value was provided.',
|
||||
) from e
|
||||
return default
|
||||
|
||||
|
||||
def match_to_find(m: re.Match):
|
||||
if m is None:
|
||||
return None
|
||||
g = m.groups(default=m.string[0:0])
|
||||
if len(g) == 1:
|
||||
return g[0]
|
||||
else:
|
||||
return g or m.group()
|
||||
|
||||
|
||||
def findalliter(pattern, string, pos=None, endpos=None, flags=0):
|
||||
"""
|
||||
like finditer(), but with return values like findall()
|
||||
|
||||
implementation taken from cpython/Modules/_sre.c/findall()
|
||||
"""
|
||||
r = (
|
||||
pattern
|
||||
if isinstance(pattern, re.Pattern)
|
||||
else re.compile(pattern, flags=flags)
|
||||
)
|
||||
if endpos is not None:
|
||||
iterator = r.finditer(string, pos=pos, endpos=endpos)
|
||||
elif pos is not None:
|
||||
iterator = r.finditer(string, pos=pos)
|
||||
else:
|
||||
iterator = r.finditer(string)
|
||||
for m in iterator:
|
||||
yield match_to_find(m)
|
||||
|
||||
|
||||
def findfirst(pattern, string, pos=None, endpos=None, flags=0, default=_undefined):
|
||||
"""
|
||||
Avoids using the inefficient findall(...)[0], or first(findall(...))
|
||||
"""
|
||||
return first(
|
||||
findalliter(pattern, string, pos=pos, endpos=endpos, flags=flags),
|
||||
default=default,
|
||||
)
|
||||
|
||||
|
||||
def topsort(nodes: Iterable[_T], order: Iterable[tuple[_T, _T]]) -> list[_T]:
|
||||
# https://en.wikipedia.org/wiki/Topological_sorting
|
||||
|
||||
order = set(order)
|
||||
result: list[_T] = [] # Empty list that will contain the sorted elements
|
||||
|
||||
def with_incoming() -> set[_T]:
|
||||
return {m for (_, m) in order}
|
||||
|
||||
# Set of all nodes with no incoming edges
|
||||
pending = list(set(nodes) - with_incoming())
|
||||
while pending:
|
||||
n = pending.pop()
|
||||
result.append(n)
|
||||
|
||||
# nodes m with an edge from n to m
|
||||
outgoing = {m for (x, m) in order if x == n}
|
||||
order -= {(n, m) for m in outgoing}
|
||||
pending.extend(outgoing - with_incoming())
|
||||
|
||||
if order:
|
||||
raise ValueError('There are cycles in the topological order')
|
||||
|
||||
return list(reversed(result)) # a topologically sorted list
|
||||
|
||||
|
||||
@cache
|
||||
def cached_re_compile(regex: re.Pattern | str | bytes) -> re.Pattern | None:
|
||||
if isinstance(regex, re.Pattern):
|
||||
return regex
|
||||
return re.compile(regex) if isinstance(regex, (str | bytes)) else None
|
||||
@@ -0,0 +1,284 @@
|
||||
import sys
|
||||
import time
|
||||
from collections import namedtuple
|
||||
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed
|
||||
from functools import partial
|
||||
from multiprocessing import Pool, cpu_count
|
||||
from pathlib import Path
|
||||
|
||||
import rich
|
||||
from rich.progress import (
|
||||
BarColumn,
|
||||
Progress,
|
||||
TaskProgressColumn,
|
||||
TextColumn,
|
||||
TimeElapsedColumn,
|
||||
TimeRemainingColumn,
|
||||
)
|
||||
|
||||
from tatsu.util.unicode_characters import (
|
||||
U_CHECK_MARK,
|
||||
U_CROSSED_SWORDS,
|
||||
)
|
||||
|
||||
from ..util import identity, memory_use, program_name, try_read
|
||||
|
||||
__all__ = ['parallel_proc', 'processing_loop']
|
||||
|
||||
|
||||
ERROR_LOG_FILENAME = 'ERROR.log'
|
||||
EOLCH = '\r' if sys.stderr.isatty() else '\n'
|
||||
sys.setrecursionlimit(2**16)
|
||||
|
||||
__Task = namedtuple('__Task', 'payload args kwargs')
|
||||
|
||||
|
||||
class ParprocResult:
|
||||
def __init__(self, payload):
|
||||
self.payload = payload
|
||||
self.outcome = None
|
||||
self.exception = None
|
||||
self.linecount = 0
|
||||
self.time = 0
|
||||
self.memory = 0
|
||||
|
||||
@property
|
||||
def success(self):
|
||||
return self.exception is None
|
||||
|
||||
def __str__(self):
|
||||
return str(self.__dict__)
|
||||
|
||||
|
||||
def process_payload(process, task, pickable=identity, reraise=False):
|
||||
start_time = time.process_time()
|
||||
result = ParprocResult(task.payload)
|
||||
try:
|
||||
outcome = process(task.payload, *task.args, **task.kwargs)
|
||||
result.memory = memory_use()
|
||||
if hasattr(outcome, 'linecount'):
|
||||
result.linecount = outcome.linecount
|
||||
else:
|
||||
result.linecount = len(try_read(task.payload).splitlines())
|
||||
result.outcome = pickable(outcome)
|
||||
except KeyboardInterrupt:
|
||||
return None
|
||||
except Exception as e:
|
||||
result.exception = e
|
||||
finally:
|
||||
result.time = time.process_time() - start_time
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _executor_pmap(executor, process, tasks):
|
||||
nworkers = max(1, cpu_count())
|
||||
n = nworkers * 8
|
||||
chunks = [tasks[i:i + n] for i in range(0, len(tasks), n)]
|
||||
for chunk in chunks:
|
||||
with executor(max_workers=nworkers) as ex:
|
||||
futures = [ex.submit(process, task) for task in chunk]
|
||||
for future in as_completed(futures):
|
||||
yield future.result()
|
||||
|
||||
|
||||
def _thread_pmap(process, tasks):
|
||||
yield from _executor_pmap(ThreadPoolExecutor, process, tasks)
|
||||
|
||||
|
||||
def _process_pmap(process, tasks):
|
||||
yield from _executor_pmap(ProcessPoolExecutor, process, tasks)
|
||||
|
||||
|
||||
def _imap_pmap(process, tasks):
|
||||
nworkers = max(1, cpu_count())
|
||||
|
||||
n = nworkers * 4
|
||||
chunks = [tasks[i:i + n] for i in range(0, len(tasks), n)]
|
||||
|
||||
count = sum(len(c) for c in chunks)
|
||||
if len(tasks) != count:
|
||||
raise RuntimeError('number of chunked tasks different %d != %d' % (len(tasks), count))
|
||||
for chunk in chunks:
|
||||
with Pool(processes=nworkers) as pool:
|
||||
yield from pool.imap_unordered(process, chunk)
|
||||
|
||||
|
||||
_pmap = _imap_pmap
|
||||
|
||||
|
||||
def parallel_proc(payloads, process, *args, **kwargs):
|
||||
pickable = kwargs.pop('pickable', identity)
|
||||
parallel = kwargs.pop('parallel', True)
|
||||
reraise = kwargs.pop('reraise', False)
|
||||
|
||||
process = partial(process_payload, process, pickable=pickable, reraise=reraise)
|
||||
tasks = [__Task(payload, args, kwargs) for payload in payloads]
|
||||
|
||||
try:
|
||||
if len(tasks) == 1:
|
||||
yield process(tasks[0])
|
||||
else:
|
||||
pmap = _pmap if parallel else map
|
||||
yield from pmap(process, tasks)
|
||||
except KeyboardInterrupt:
|
||||
return
|
||||
|
||||
|
||||
def _build_progressbar(total):
|
||||
progress = Progress(
|
||||
TextColumn(f"[progress.description]{program_name()}"),
|
||||
BarColumn(),
|
||||
# *Progress.get_default_columns(),
|
||||
TaskProgressColumn(),
|
||||
TimeElapsedColumn(),
|
||||
TimeRemainingColumn(),
|
||||
TextColumn("[progress.description]{task.description}"),
|
||||
refresh_per_second=1,
|
||||
speed_estimate_period=30.0,
|
||||
)
|
||||
task = progress.add_task('', total=total)
|
||||
return progress, task
|
||||
|
||||
|
||||
def processing_loop(filenames, process, *args, reraise=False, **kwargs): # pylint: disable=too-many-locals
|
||||
try:
|
||||
total = len(filenames)
|
||||
total_time = 0
|
||||
run_time = 0
|
||||
start_time = time.time()
|
||||
results = parallel_proc(filenames, process, *args, **kwargs)
|
||||
results = results or []
|
||||
count = 0
|
||||
success_count = 0
|
||||
success_linecount = 0
|
||||
progress, progress_task = _build_progressbar(total)
|
||||
|
||||
if total == 1:
|
||||
log = sys.stderr
|
||||
else:
|
||||
log = Path(ERROR_LOG_FILENAME).open('w')
|
||||
|
||||
with progress:
|
||||
for result in results:
|
||||
if result is None:
|
||||
continue
|
||||
count += 1
|
||||
|
||||
total_time = time.time() - start_time
|
||||
filename = Path(result.payload).name
|
||||
if result.exception:
|
||||
icon = f'[red]{U_CROSSED_SWORDS}'
|
||||
else:
|
||||
icon = f'[green]{U_CHECK_MARK}'
|
||||
|
||||
progress.update(
|
||||
progress_task,
|
||||
advance=1,
|
||||
description=f'{icon} {filename}',
|
||||
)
|
||||
|
||||
if result.exception:
|
||||
print(file=log)
|
||||
try:
|
||||
print('ERROR:', result.payload, file=log)
|
||||
print(result.exception, file=log)
|
||||
except Exception:
|
||||
# in case of errors while serializing the exception
|
||||
print('EXCEPTION', type(result.exception).__name__, file=log)
|
||||
if reraise:
|
||||
raise result.exception
|
||||
elif result.outcome is not None:
|
||||
success_count += 1
|
||||
success_linecount += result.linecount
|
||||
run_time += result.time
|
||||
yield result
|
||||
|
||||
log.flush()
|
||||
|
||||
progress.update(progress_task, advance=0, description='')
|
||||
progress.stop()
|
||||
file_process_summary(
|
||||
filenames, total_time, run_time, success_count, success_linecount, log,
|
||||
)
|
||||
except KeyboardInterrupt:
|
||||
return
|
||||
|
||||
|
||||
def file_process_progress(latest_result, count, total, total_time):
|
||||
filename = latest_result.payload
|
||||
|
||||
percent = count / total
|
||||
mb_memory = (latest_result.memory + memory_use()) // (1024 * 1024)
|
||||
|
||||
eta = (total - count) * 0.8 * total_time / (0.2 * count)
|
||||
bar = '[%-24s]' % ('#' * round(24 * percent))
|
||||
|
||||
print(
|
||||
'%3d/%-3d' % (count, total),
|
||||
bar,
|
||||
'%3d%%' % (100 * percent),
|
||||
# format_hours(total_time),
|
||||
f'{format_hours(eta)}ETA',
|
||||
format_minutes(latest_result),
|
||||
'%3dMiB' % mb_memory if mb_memory else '',
|
||||
(Path(filename).name + ' ' * 80)[:40],
|
||||
file=sys.stderr,
|
||||
end=EOLCH)
|
||||
|
||||
|
||||
def format_minutes(result):
|
||||
return f'{result.time / 60:3.0f}:{result.time % 60:04.1f}'
|
||||
|
||||
|
||||
def format_hours(time):
|
||||
return f'{time // 3600:2.0f}:{(time // 60) % 60:02.0f}:{time % 60:02.0f}'
|
||||
|
||||
|
||||
def file_process_summary(
|
||||
filenames, total_time, run_time, success_count, success_linecount, log,
|
||||
):
|
||||
filecount = 0
|
||||
linecount = 0
|
||||
for fname in filenames:
|
||||
filecount += 1
|
||||
|
||||
nlines = len(try_read(fname).splitlines())
|
||||
linecount += nlines
|
||||
|
||||
failures = filecount - success_count
|
||||
|
||||
summary_text = '''\
|
||||
-----------------------------------------------------------------------
|
||||
{:12,d} files input
|
||||
{:12,d} source lines input
|
||||
{:12,d} total lines processed
|
||||
{:12,d} successes
|
||||
{:12,d} failures
|
||||
{:12.1f}% success rate
|
||||
{:>13s} time
|
||||
{:>13s} run time
|
||||
'''
|
||||
summary_text = '\n'.join(s.strip() for s in summary_text.splitlines())
|
||||
|
||||
summary = summary_text.format(
|
||||
filecount,
|
||||
linecount,
|
||||
success_linecount,
|
||||
success_count,
|
||||
failures,
|
||||
100 * success_count / filecount if filecount != 0 else 0,
|
||||
format_hours(total_time),
|
||||
format_hours(run_time),
|
||||
)
|
||||
print(EOLCH + 80 * ' ', file=log)
|
||||
print(summary, file=log)
|
||||
print(EOLCH + 80 * ' ', file=sys.stderr)
|
||||
|
||||
if failures:
|
||||
rich.print(f'[red bold]FAILURES logged to [green]{log.name}!')
|
||||
print(file=sys.stderr)
|
||||
if log != sys.stderr:
|
||||
print(summary, file=sys.stderr)
|
||||
if failures:
|
||||
sys.exit(1)
|
||||
@@ -0,0 +1,85 @@
|
||||
import argparse
|
||||
|
||||
from . import filelist_from_patterns
|
||||
from .parproc import processing_loop
|
||||
|
||||
|
||||
def parallel_test_run(parse, options):
|
||||
def pysearch(pattern):
|
||||
if pattern.endswith('.py'):
|
||||
return pattern
|
||||
else:
|
||||
if not pattern.endswith('/'):
|
||||
pattern += '/'
|
||||
if '**' not in pattern:
|
||||
pattern += '**/'
|
||||
return pattern + '*.py'
|
||||
|
||||
try:
|
||||
patterns = [pysearch(p) for p in options.patterns]
|
||||
filenames = filelist_from_patterns(
|
||||
patterns, sizesort=options.sort, ignore=options.ignore,
|
||||
)
|
||||
|
||||
kwargs = vars(options)
|
||||
kwargs.pop('patterns', None)
|
||||
kwargs.pop('sort', None)
|
||||
kwargs.pop('ignore', None)
|
||||
parallel = not kwargs.pop('serial', False)
|
||||
|
||||
return processing_loop(parse, filenames, parallel=parallel, **kwargs)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
if options.verbose:
|
||||
raise
|
||||
|
||||
|
||||
def parse_args():
|
||||
argparser = argparse.ArgumentParser(
|
||||
prog=__package__,
|
||||
# description=DESCRIPTION,
|
||||
add_help=False,
|
||||
)
|
||||
|
||||
argparser.add_argument(
|
||||
'patterns', nargs='+', metavar='PATTERNS', help='filename patterns',
|
||||
)
|
||||
argparser.add_argument(
|
||||
'--help', '-h', help='show this help message and exit', action='help',
|
||||
)
|
||||
argparser.add_argument(
|
||||
'--ignore',
|
||||
'-i',
|
||||
metavar='PATTERN',
|
||||
help='ignore these patterns',
|
||||
action='append',
|
||||
)
|
||||
argparser.add_argument(
|
||||
'--sort', '-s', help='sort files by size', action='store_true',
|
||||
)
|
||||
argparser.add_argument(
|
||||
'--serial', '-S', help='do not run in parallel', action='store_true',
|
||||
)
|
||||
argparser.add_argument(
|
||||
'--trace',
|
||||
'-t',
|
||||
help='produce verbose output for a parse',
|
||||
action='store_true',
|
||||
)
|
||||
argparser.add_argument(
|
||||
'--verbose',
|
||||
'-v',
|
||||
help='output exceptions as they happen',
|
||||
action='store_true',
|
||||
)
|
||||
argparser.add_argument(
|
||||
'--exitfirst',
|
||||
'-x',
|
||||
help='output exceptions as they happen',
|
||||
action='store_true',
|
||||
)
|
||||
return argparser.parse_args()
|
||||
|
||||
|
||||
def generic_main(parse):
|
||||
return parallel_test_run(parse, parse_args())
|
||||
@@ -0,0 +1,54 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from tatsu.util import is_posix
|
||||
|
||||
U_LARROW = '\u2190'
|
||||
U_DLARROW = '\u2199'
|
||||
U_LDARROW = '\u21D0'
|
||||
U_UDARROW = '\u21D1'
|
||||
U_RDARROW = '\u21D2'
|
||||
U_DDARROW = '\u21D3'
|
||||
U_L_TRIPPLE_ARROW = '\u21DA'
|
||||
|
||||
U_WARNING = '\u26A0'
|
||||
|
||||
U_NOT_EQUAL_TO = '\u2260'
|
||||
U_IDENTICAL_TO = '\u2261'
|
||||
U_NOT_IDENTICAL_TO = '\u2262'
|
||||
|
||||
U_CHECK_MARK = '\u2713'
|
||||
|
||||
U_POWER_SYMBOL = '\u23FB'
|
||||
U_POWER_ON_SYMBOL = '\u23FC'
|
||||
U_POWER_OFF_SYMBOL = '\u23FD'
|
||||
|
||||
U_GREEK_SMALL_LETTER_EPSILON = '\u03B5'
|
||||
U_REGISTERED_SIGN = '\u00AE'
|
||||
U_RIENNMAN = '\u211D'
|
||||
|
||||
U_ANTICLOCKWISE_OPEN_CIRCLE_ARROW = '\u21BA'
|
||||
U_ANTICLOCKWISE_GAPPED_CIRCLE_ARROW = '\u27F2'
|
||||
|
||||
U_PUNCTUATION_SPACE = '\u2008'
|
||||
U_FOUR_PER_EM_SPACE = '\u2005'
|
||||
U_MEDIUM_MATHEMATICAL_SPACE = '\u205F'
|
||||
U_ZERO_WIDTH_NO_BREAK_SPACE = '\uFEFF'
|
||||
|
||||
U_BLACK_SCISSORS = '\u2702'
|
||||
U_CROSSED_SWORDS = '\u2694'
|
||||
|
||||
|
||||
if not is_posix():
|
||||
C_DERIVE = '<'
|
||||
C_ENTRY = '<'
|
||||
C_SUCCESS = '>'
|
||||
C_FAILURE = '!'
|
||||
C_RECURSION = 'r '
|
||||
C_CUT = '~'
|
||||
else:
|
||||
C_DERIVE = U_DLARROW
|
||||
C_ENTRY = C_DERIVE
|
||||
C_SUCCESS = U_IDENTICAL_TO
|
||||
C_FAILURE = U_NOT_IDENTICAL_TO
|
||||
C_RECURSION = U_ANTICLOCKWISE_GAPPED_CIRCLE_ARROW + U_FOUR_PER_EM_SPACE
|
||||
C_CUT = U_CROSSED_SWORDS
|
||||
@@ -0,0 +1,174 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Mapping
|
||||
from contextlib import contextmanager
|
||||
from typing import Any
|
||||
|
||||
from .objectmodel import Node
|
||||
from .util import is_list
|
||||
|
||||
|
||||
class NodeWalkerMeta(type):
|
||||
def __new__(mcs, name, bases, dct):
|
||||
class_ = super().__new__(mcs, name, bases, dct)
|
||||
class_._walker_cache = {}
|
||||
return class_
|
||||
|
||||
|
||||
class NodeWalker(metaclass=NodeWalkerMeta):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
# copy the class attribute to avoid linter warnings
|
||||
self._walker_cache = type(
|
||||
self,
|
||||
)._walker_cache # pylint: disable=no-member
|
||||
|
||||
def walk(self, node: Node | list[Node], *args, **kwargs) -> Any:
|
||||
if isinstance(node, list | tuple):
|
||||
return [self.walk(n, *args, **kwargs) for n in node]
|
||||
|
||||
if isinstance(node, Mapping):
|
||||
return {
|
||||
name: self.walk(value, *args, **kwargs)
|
||||
for name, value in node.items()
|
||||
}
|
||||
|
||||
walker = self._find_walker(node)
|
||||
if callable(walker):
|
||||
return walker(self, node, *args, **kwargs)
|
||||
else:
|
||||
return node
|
||||
|
||||
def walk_children(self, node: Node, *args, **kwargs):
|
||||
if not isinstance(node, Node):
|
||||
return []
|
||||
|
||||
return [
|
||||
self.walk(child, *args, **kwargs)
|
||||
for child in node.children()
|
||||
]
|
||||
|
||||
# note: backwards compatibility
|
||||
_walk_children = walk_children
|
||||
|
||||
def _find_walker(self, node: Node, prefix='walk_'):
|
||||
def pythonize_match(m):
|
||||
return '_' + m.group().lower()
|
||||
|
||||
cls = self.__class__
|
||||
node_cls = node.__class__
|
||||
node_cls_qualname = node_cls.__qualname__
|
||||
|
||||
if walker := self._walker_cache.get(node_cls_qualname):
|
||||
return walker
|
||||
|
||||
node_classes = [node.__class__]
|
||||
while node_classes:
|
||||
node_cls = node_classes.pop(0)
|
||||
|
||||
cammelcase_name = node_cls.__name__
|
||||
walker = getattr(cls, prefix + cammelcase_name, None)
|
||||
if callable(walker):
|
||||
break
|
||||
|
||||
# walk__pythonic_name with double underscore after walk
|
||||
pythonic_name = re.sub(
|
||||
r'[A-Z]+', pythonize_match, node_cls.__name__,
|
||||
)
|
||||
if pythonic_name != cammelcase_name:
|
||||
walker = getattr(cls, prefix + pythonic_name, None)
|
||||
if callable(walker):
|
||||
break
|
||||
|
||||
# walk_pythonic_name with single underscore after prefix
|
||||
pythonic_name = pythonic_name.lstrip('_')
|
||||
if pythonic_name != cammelcase_name:
|
||||
walker = getattr(cls, prefix + pythonic_name, None)
|
||||
if callable(walker):
|
||||
break
|
||||
|
||||
for b in node_cls.__bases__:
|
||||
if b not in node_classes:
|
||||
node_classes.append(b)
|
||||
else:
|
||||
walker = getattr(cls, '_walk_default', None)
|
||||
if walker is None:
|
||||
walker = getattr(
|
||||
cls, 'walk_default', None,
|
||||
) # backwards compatibility
|
||||
if not callable(walker):
|
||||
walker = None
|
||||
|
||||
self._walker_cache[node_cls_qualname] = walker
|
||||
return walker
|
||||
|
||||
|
||||
class PreOrderWalker(NodeWalker):
|
||||
def walk(self, node, *args, **kwargs):
|
||||
result = super().walk(node, *args, **kwargs)
|
||||
if result is not None:
|
||||
self.walk_children(node, *args, **kwargs)
|
||||
return result
|
||||
|
||||
|
||||
class DepthFirstWalker(NodeWalker):
|
||||
def walk(self, node, *args, **kwargs):
|
||||
if isinstance(node, Node):
|
||||
children = [self.walk(c, *args, **kwargs) for c in node.children()]
|
||||
return super().walk(node, children, *args, **kwargs)
|
||||
elif isinstance(node, Mapping):
|
||||
return {n: self.walk(e, *args, **kwargs) for n, e in node.items()}
|
||||
elif is_list(node):
|
||||
return [self.walk(e, *args, **kwargs) for e in iter(node)]
|
||||
else:
|
||||
return super().walk(node, [], *args, **kwargs)
|
||||
|
||||
|
||||
class ContextWalker(NodeWalker):
|
||||
def __init__(self, initial_context):
|
||||
super().__init__()
|
||||
self._initial_context = initial_context
|
||||
self._context_stack = [initial_context]
|
||||
|
||||
# abstract
|
||||
def get_node_context(self, node, *args, **kwargs):
|
||||
return node
|
||||
|
||||
# abstract
|
||||
def enter_context(self, ctx):
|
||||
pass
|
||||
|
||||
# abstract
|
||||
def leave_context(self, ctx):
|
||||
pass
|
||||
|
||||
def push_context(self, ctx):
|
||||
self._context_stack.append(ctx)
|
||||
|
||||
def pop_context(self):
|
||||
self._context_stack.pop()
|
||||
|
||||
@property
|
||||
def initial_context(self):
|
||||
return self._initial_context
|
||||
|
||||
@property
|
||||
def context(self):
|
||||
return self._context_stack[-1]
|
||||
|
||||
@contextmanager
|
||||
def new_context(self, node, *args, **kwargs):
|
||||
ctx = self.get_node_context(node, *args, **kwargs)
|
||||
if ctx == self.context:
|
||||
yield ctx
|
||||
else:
|
||||
self.enter_context(ctx)
|
||||
try:
|
||||
self.push_context(ctx)
|
||||
try:
|
||||
yield ctx
|
||||
finally:
|
||||
self.pop_context()
|
||||
finally:
|
||||
self.leave_context(ctx)
|
||||
Reference in New Issue
Block a user