1101 lines
28 KiB
Python
1101 lines
28 KiB
Python
from __future__ import annotations
|
|
|
|
import functools
|
|
from collections import defaultdict
|
|
from collections.abc import Mapping
|
|
from copy import copy
|
|
from itertools import takewhile
|
|
from pathlib import Path
|
|
|
|
from .ast import AST
|
|
from .collections import OrderedSet as oset
|
|
from .contexts import ParseContext
|
|
from .exceptions import FailedRef, GrammarError
|
|
from .infos import ParserConfig, RuleInfo
|
|
from .leftrec import Nullable, find_left_recursion
|
|
from .objectmodel import Node
|
|
from .util import chunks, compress_seq, indent, re, trim
|
|
|
|
PEP8_LLEN = 72
|
|
|
|
|
|
PRAGMA_RE = r'^\s*#include.*$'
|
|
|
|
|
|
class _ref(str):
|
|
def __repr__(self):
|
|
return f'<{self}>'
|
|
|
|
|
|
def ref(name):
|
|
return (_ref(name),)
|
|
|
|
|
|
def kdot(x, y, k):
|
|
if not y:
|
|
return oset(a[:k] for a in x)
|
|
elif not x:
|
|
return oset(b[:k] for b in y)
|
|
else:
|
|
return oset((a + b)[:k] for a in x for b in y)
|
|
|
|
|
|
def pythonize_name(name):
|
|
return ''.join('_' + c.lower() if c.isupper() else c for c in name)
|
|
|
|
|
|
class ModelContext(ParseContext):
|
|
def __init__(
|
|
self,
|
|
rules,
|
|
/,
|
|
start=None,
|
|
config: ParserConfig | None = None,
|
|
**settings,
|
|
):
|
|
config = ParserConfig.new(config, **settings)
|
|
config = config.replace(start=start)
|
|
|
|
super().__init__(config=config)
|
|
|
|
self.rules = {rule.name: rule for rule in rules}
|
|
|
|
@property
|
|
def pos(self):
|
|
return self._tokenizer.pos
|
|
|
|
def _find_rule(self, name):
|
|
return functools.partial(self.rules[name].parse, self)
|
|
|
|
|
|
class Model(Node):
|
|
@staticmethod
|
|
def classes():
|
|
return [
|
|
c
|
|
for c in globals().values()
|
|
if isinstance(c, type) and issubclass(c, Model)
|
|
]
|
|
|
|
def __init__(self, ast=None, ctx=None):
|
|
super().__init__(ast=ast, ctx=ctx)
|
|
self._lookahead = None
|
|
self._firstset = None
|
|
self._follow_set = oset()
|
|
self.value = None
|
|
self._nullability = self._nullable()
|
|
if isinstance(self._nullability, int): # Allow simple boolean values
|
|
if self._nullability:
|
|
self._nullability = Nullable.yes()
|
|
else:
|
|
self._nullability = Nullable.no()
|
|
|
|
def parse(self, ctx):
|
|
ctx.last_node = None
|
|
|
|
def defines(self):
|
|
return []
|
|
|
|
def _add_defined_attributes(self, ctx, ast=None):
|
|
if ast is None:
|
|
return
|
|
if not hasattr(ast, '_define'):
|
|
return
|
|
|
|
defines = dict(compress_seq(self.defines()))
|
|
|
|
keys = [k for k, list in defines.items() if not list]
|
|
list_keys = [k for k, list in defines.items() if list]
|
|
ctx._define(keys, list_keys)
|
|
ast._define(keys, list_keys)
|
|
|
|
def lookahead(self, k=1):
|
|
if self._lookahead is None:
|
|
self._lookahead = kdot(self.firstset(k), self.followset(k), k)
|
|
return self._lookahead
|
|
|
|
def lookahead_str(self):
|
|
return ' '.join(sorted(repr(f[0]) for f in self.lookahead() if f))
|
|
|
|
def firstset(self, k=1):
|
|
if self._firstset is None:
|
|
self._firstset = self._first(k, defaultdict(oset))
|
|
return self._firstset
|
|
|
|
def followset(self, k=1):
|
|
return self._follow_set
|
|
|
|
def missing_rules(self, rules):
|
|
return oset()
|
|
|
|
def _used_rule_names(self):
|
|
return oset()
|
|
|
|
def _first(self, k, f):
|
|
return oset()
|
|
|
|
def _follow(self, k, fl, a):
|
|
return a
|
|
|
|
def is_nullable(self, ctx=None):
|
|
return self._nullability.nullable
|
|
|
|
def _nullable(self):
|
|
return False
|
|
|
|
# list of rules that can be invoked at the same position
|
|
def at_same_pos(self, ctx):
|
|
return []
|
|
|
|
def comments_str(self):
|
|
comments, _eol = self.comments
|
|
if not comments:
|
|
return ''
|
|
|
|
return '\n'.join(
|
|
'(* {} *)\n'.format('\n'.join(c).replace('(*', '').replace('*)', '').strip())
|
|
for c in comments
|
|
)
|
|
|
|
def nodecount(self):
|
|
return 1
|
|
|
|
def pretty(self):
|
|
return self._to_str()
|
|
|
|
def pretty_lean(self):
|
|
return self._to_str(lean=True)
|
|
|
|
def _to_str(self, lean=False):
|
|
return '%s:%d' % (type(self).__name__, id(self))
|
|
|
|
def __str__(self):
|
|
return self._to_str()
|
|
|
|
|
|
class Void(Model):
|
|
def parse(self, ctx):
|
|
return ctx._void()
|
|
|
|
def _to_str(self, lean=False):
|
|
return '()'
|
|
|
|
def _nullable(self):
|
|
return True
|
|
|
|
|
|
class Any(Model):
|
|
def parse(self, ctx):
|
|
return ctx._any()
|
|
|
|
def _to_str(self, lean=False):
|
|
return '/./'
|
|
|
|
|
|
class Fail(Model):
|
|
def parse(self, ctx):
|
|
return ctx._fail()
|
|
|
|
def _to_str(self, lean=False):
|
|
return '!()'
|
|
|
|
|
|
class Comment(Model):
|
|
def __init__(self, ast=None, **kwargs):
|
|
self.comment = None
|
|
super().__init__(ast=AST(comment=ast))
|
|
|
|
def _to_str(self, lean=False):
|
|
return f'(* {self.comment} *)'
|
|
|
|
|
|
class EOLComment(Comment):
|
|
def _to_str(self, lean=False):
|
|
return f' # {self.comment}\n'
|
|
|
|
|
|
class EOF(Model):
|
|
def parse(self, ctx):
|
|
ctx._check_eof()
|
|
|
|
def _to_str(self, lean=False):
|
|
return '$'
|
|
|
|
|
|
class Decorator(Model):
|
|
def __init__(self, ast=None, exp=None, **kwargs):
|
|
if exp is not None:
|
|
self.exp = ast = exp
|
|
elif not isinstance(ast, AST):
|
|
# Patch to avoid bad interactions with attribute setting in Model.
|
|
# Also a shortcut for subexpressions that are not ASTs.
|
|
ast = AST(exp=ast)
|
|
super().__init__(ast=ast)
|
|
assert isinstance(self.exp, Model)
|
|
|
|
def parse(self, ctx):
|
|
return self.exp.parse(ctx)
|
|
|
|
def defines(self):
|
|
return self.exp.defines()
|
|
|
|
def missing_rules(self, rules):
|
|
return self.exp.missing_rules(rules)
|
|
|
|
def _used_rule_names(self):
|
|
return self.exp._used_rule_names()
|
|
|
|
def _first(self, k, f):
|
|
return self.exp._first(k, f)
|
|
|
|
def _follow(self, k, fl, a):
|
|
return self.exp._follow(k, fl, a)
|
|
|
|
def nodecount(self):
|
|
return 1 + self.exp.nodecount()
|
|
|
|
def _to_str(self, lean=False):
|
|
return self.exp._to_str(lean=lean)
|
|
|
|
def _nullable(self):
|
|
return Nullable.of(self.exp)
|
|
|
|
def at_same_pos(self, ctx):
|
|
return [self.exp]
|
|
|
|
|
|
# NOTE: backwards compatibility
|
|
_Decorator = Decorator
|
|
|
|
|
|
class Group(Decorator):
|
|
def parse(self, ctx):
|
|
with ctx._group():
|
|
self.exp.parse(ctx)
|
|
return ctx.last_node
|
|
|
|
def _to_str(self, lean=False):
|
|
exp = self.exp._to_str(lean=lean)
|
|
if len(exp.splitlines()) > 1:
|
|
return f'(\n{indent(exp)}\n)'
|
|
else:
|
|
return f'({trim(exp)})'
|
|
|
|
|
|
class Token(Model):
|
|
def __post_init__(self):
|
|
super().__post_init__()
|
|
ast = self.ast
|
|
self.token = ast
|
|
|
|
def parse(self, ctx):
|
|
return ctx._token(self.token)
|
|
|
|
def _first(self, k, f):
|
|
return {(self.token,)}
|
|
|
|
def _to_str(self, lean=False):
|
|
return repr(self.token)
|
|
|
|
|
|
class Constant(Model):
|
|
def __post_init__(self):
|
|
super().__post_init__()
|
|
self.literal = self.ast
|
|
|
|
def parse(self, ctx):
|
|
return ctx._constant(self.literal)
|
|
|
|
def _first(self, k, f):
|
|
return {()}
|
|
|
|
def _to_str(self, lean=False):
|
|
return f'`{self.literal!r}`'
|
|
|
|
def _nullable(self):
|
|
return True
|
|
|
|
|
|
class Alert(Constant):
|
|
def __post_init__(self):
|
|
super().__post_init__()
|
|
self.literal = self.ast.message.literal
|
|
self.level = len(self.ast.level)
|
|
|
|
def parse(self, ctx):
|
|
return super().parse(ctx)
|
|
|
|
def _to_str(self, lean=False):
|
|
return f'{"^" * self.level}{super()._to_str()}'
|
|
|
|
|
|
class Pattern(Model):
|
|
def __post_init__(self):
|
|
super().__post_init__()
|
|
ast = self.ast
|
|
if not isinstance(ast, list):
|
|
ast = [ast]
|
|
self.patterns = ast
|
|
self.regex = re.compile(self.pattern)
|
|
|
|
@property
|
|
def pattern(self):
|
|
return ''.join(self.patterns)
|
|
|
|
def parse(self, ctx):
|
|
return ctx._pattern(self.pattern)
|
|
|
|
def _first(self, k, f):
|
|
x = self
|
|
if bool(self.regex.match('')):
|
|
return oset([(), (x,)])
|
|
else:
|
|
return {(x,)}
|
|
|
|
def _to_str(self, lean=False):
|
|
parts = []
|
|
for pat in (str(p) for p in self.patterns):
|
|
if '/' in pat:
|
|
newpat = pat.replace('"', r'\"')
|
|
regex = f'?"{newpat}"'
|
|
else:
|
|
regex = f'/{pat}/'
|
|
parts.append(regex)
|
|
return '\n+ '.join(parts)
|
|
|
|
def _nullable(self):
|
|
return bool(self.regex.match(''))
|
|
|
|
def __repr__(self):
|
|
return self.pattern.replace('\\\\', '\\')
|
|
|
|
|
|
class Lookahead(Decorator):
|
|
def parse(self, ctx):
|
|
with ctx._if():
|
|
return super().parse(ctx)
|
|
|
|
def _to_str(self, lean=False):
|
|
return '&' + self.exp._to_str(lean=lean)
|
|
|
|
def _nullable(self):
|
|
return True
|
|
|
|
|
|
class NegativeLookahead(Decorator):
|
|
def parse(self, ctx):
|
|
with ctx._ifnot():
|
|
return super().parse(ctx)
|
|
|
|
def _to_str(self, lean=False):
|
|
return '!' + str(self.exp._to_str(lean=lean))
|
|
|
|
def _nullable(self):
|
|
return True
|
|
|
|
|
|
class SkipTo(Decorator):
|
|
def parse(self, ctx):
|
|
super_parse = super().parse
|
|
return ctx._skip_to(lambda: super_parse(ctx))
|
|
|
|
def _first(self, k, f):
|
|
# use None to represent ANY
|
|
return oset({(None,)}) | super()._first(k, f)
|
|
|
|
def _to_str(self, lean=False):
|
|
return '->' + self.exp._to_str(lean=lean)
|
|
|
|
|
|
class Sequence(Model):
|
|
def __init__(self, ast, **kwargs):
|
|
assert ast.sequence
|
|
self.sequence = ()
|
|
super().__init__(ast=ast)
|
|
|
|
def parse(self, ctx):
|
|
ctx.last_node = [s.parse(ctx) for s in self.sequence]
|
|
return ctx.last_node
|
|
|
|
def defines(self):
|
|
return [d for s in self.sequence for d in s.defines()]
|
|
|
|
def missing_rules(self, rules):
|
|
return oset().union(*[s.missing_rules(rules) for s in self.sequence])
|
|
|
|
def _used_rule_names(self):
|
|
return oset().union(*[s._used_rule_names() for s in self.sequence])
|
|
|
|
def _first(self, k, f):
|
|
result = {()}
|
|
for s in self.sequence:
|
|
x = s._first(k, f)
|
|
# FIXME:
|
|
# if isinstance(x, RuleRef):
|
|
# x |= f[x.name]
|
|
result = kdot(result, x, k)
|
|
self._firstset = result
|
|
return result
|
|
|
|
def _follow(self, k, fl, a):
|
|
fs = a
|
|
for x in reversed(self.sequence):
|
|
if isinstance(x, RuleRef):
|
|
fl[x.name] |= fs
|
|
x._follow(k, fl, fs)
|
|
fs = kdot(x.firstset(k=k), fs, k)
|
|
return a
|
|
|
|
def nodecount(self):
|
|
return 1 + sum(s.nodecount() for s in self.sequence)
|
|
|
|
def _to_str(self, lean=False):
|
|
comments = self.comments_str()
|
|
seq = [str(s._to_str(lean=lean)) for s in self.sequence]
|
|
single = ' '.join(seq)
|
|
if len(single) <= PEP8_LLEN and len(single.splitlines()) <= 1:
|
|
return comments + single
|
|
else:
|
|
return comments + '\n'.join(seq)
|
|
|
|
def _nullable(self):
|
|
return Nullable.all(self.sequence)
|
|
|
|
def at_same_pos(self, ctx):
|
|
head = list(takewhile(lambda c: c.is_nullable(ctx), self.sequence))
|
|
if len(head) < len(self.sequence):
|
|
head.append(self.sequence[len(head)])
|
|
return head
|
|
|
|
|
|
class Choice(Model):
|
|
def __init__(self, ast=None, **kwargs):
|
|
self.options = []
|
|
super().__init__(ast=AST(options=ast))
|
|
assert isinstance(self.options, list), repr(self.options)
|
|
|
|
def parse(self, ctx):
|
|
with ctx._choice():
|
|
for o in self.options:
|
|
with ctx._option():
|
|
ctx.last_node = o.parse(ctx)
|
|
return ctx.last_node
|
|
|
|
lookahead = self.lookahead_str()
|
|
if lookahead:
|
|
ctx._error(f'expecting one of: {lookahead}:')
|
|
ctx._error('no available options')
|
|
return None
|
|
|
|
def defines(self):
|
|
return [d for o in self.options for d in o.defines()]
|
|
|
|
def missing_rules(self, rules):
|
|
return oset().union(*[o.missing_rules(rules) for o in self.options])
|
|
|
|
def _used_rule_names(self):
|
|
return oset().union(*[o._used_rule_names() for o in self.options])
|
|
|
|
def _first(self, k, f):
|
|
result = oset()
|
|
for o in self.options:
|
|
result |= o._first(k, f)
|
|
self._firstset = result
|
|
return result
|
|
|
|
def _follow(self, k, fl, a):
|
|
for o in self.options:
|
|
o._follow(k, fl, a)
|
|
return a
|
|
|
|
def nodecount(self):
|
|
return 1 + sum(o.nodecount() for o in self.options)
|
|
|
|
def _to_str(self, lean=False):
|
|
options = [str(o._to_str(lean=lean)) for o in self.options]
|
|
|
|
multi = any(len(o.splitlines()) > 1 for o in options)
|
|
single = ' | '.join(o for o in options)
|
|
|
|
if multi:
|
|
return '\n|\n'.join(indent(o) for o in options)
|
|
elif options and len(single) > PEP8_LLEN:
|
|
return '| ' + '\n| '.join(o for o in options)
|
|
else:
|
|
return single
|
|
|
|
def _nullable(self):
|
|
return Nullable.any(self.options)
|
|
|
|
def at_same_pos(self, ctx):
|
|
return self.options
|
|
|
|
|
|
class Option(Decorator):
|
|
def parse(self, ctx):
|
|
result = super().parse(ctx)
|
|
self._add_defined_attributes(ctx, result)
|
|
return result
|
|
|
|
|
|
class Closure(Decorator):
|
|
def parse(self, ctx):
|
|
return ctx._closure(lambda: self.exp.parse(ctx))
|
|
|
|
def _first(self, k, f):
|
|
efirst = self.exp._first(k, f)
|
|
result = {()}
|
|
for _i in range(k):
|
|
result = kdot(result, efirst, k)
|
|
return oset({()}) | result
|
|
|
|
def _to_str(self, lean=False):
|
|
sexp = str(self.exp._to_str(lean=lean))
|
|
if len(sexp.splitlines()) <= 1:
|
|
return f'{{{sexp}}}'
|
|
else:
|
|
return f'{{\n{indent(sexp)}\n}}'
|
|
|
|
def _nullable(self):
|
|
return True
|
|
|
|
|
|
class PositiveClosure(Closure):
|
|
def parse(self, ctx):
|
|
return ctx._positive_closure(lambda: self.exp.parse(ctx))
|
|
|
|
def _first(self, k, f):
|
|
efirst = self.exp._first(k, f)
|
|
result = {()}
|
|
for _i in range(k):
|
|
result = kdot(result, efirst, k)
|
|
return result
|
|
|
|
def _to_str(self, lean=False):
|
|
return super()._to_str(lean=lean) + '+'
|
|
|
|
def _nullable(self):
|
|
return Nullable.of(self.exp)
|
|
|
|
|
|
class Join(Decorator):
|
|
JOINOP = '%'
|
|
|
|
def __init__(self, ast=None, **kwargs):
|
|
super().__init__(ast.exp)
|
|
self.sep = ast.sep
|
|
|
|
def parse(self, ctx):
|
|
def sep():
|
|
return self.sep.parse(ctx)
|
|
|
|
def exp():
|
|
return self.exp.parse(ctx)
|
|
|
|
return self._do_parse(ctx, exp, sep)
|
|
|
|
def _do_parse(self, ctx, exp, sep):
|
|
return ctx._join(exp, sep)
|
|
|
|
def _to_str(self, lean=False):
|
|
ssep = self.sep._to_str(lean=lean)
|
|
sexp = str(self.exp._to_str(lean=lean))
|
|
if len(sexp.splitlines()) <= 1:
|
|
return f'{ssep}{self.JOINOP}{{{sexp}}}'
|
|
else:
|
|
return f'{ssep}{self.JOINOP}{{\n{sexp}\n}}'
|
|
|
|
def _nullable(self):
|
|
return True
|
|
|
|
|
|
class PositiveJoin(Join):
|
|
def _do_parse(self, ctx, exp, sep):
|
|
return ctx._positive_join(exp, sep)
|
|
|
|
def _to_str(self, lean=False):
|
|
return super()._to_str(lean=lean) + '+'
|
|
|
|
def _nullable(self):
|
|
return Nullable.of(self.exp)
|
|
|
|
|
|
class LeftJoin(PositiveJoin):
|
|
JOINOP = '<'
|
|
|
|
def _do_parse(self, ctx, exp, sep):
|
|
return ctx._left_join(exp, sep)
|
|
|
|
|
|
class RightJoin(PositiveJoin):
|
|
JOINOP = '>'
|
|
|
|
def _do_parse(self, ctx, exp, sep):
|
|
return ctx._right_join(exp, sep)
|
|
|
|
|
|
class Gather(Join):
|
|
JOINOP = '.'
|
|
|
|
def _do_parse(self, ctx, exp, sep):
|
|
return ctx._gather(exp, sep)
|
|
|
|
|
|
class PositiveGather(Gather):
|
|
def _do_parse(self, ctx, exp, sep):
|
|
return ctx._positive_gather(exp, sep)
|
|
|
|
def _to_str(self, lean=False):
|
|
return super()._to_str(lean=lean) + '+'
|
|
|
|
def _nullable(self):
|
|
return Nullable.of(self.exp)
|
|
|
|
|
|
class EmptyClosure(Model):
|
|
def parse(self, ctx):
|
|
return ctx._empty_closure()
|
|
|
|
def _first(self, k, f):
|
|
return {()}
|
|
|
|
def _to_str(self, lean=False):
|
|
return '{}'
|
|
|
|
def _nullable(self):
|
|
return True
|
|
|
|
|
|
class Optional(Decorator):
|
|
def parse(self, ctx):
|
|
ctx.last_node = None
|
|
self._add_defined_attributes(ctx, ctx.ast)
|
|
with ctx._optional():
|
|
return self.exp.parse(ctx)
|
|
|
|
def _first(self, k, f):
|
|
return oset({()}) | self.exp._first(k, f)
|
|
|
|
def _to_str(self, lean=False):
|
|
exp = str(self.exp._to_str(lean=lean))
|
|
template = '[%s]'
|
|
if isinstance(self.exp, Choice):
|
|
template = trim(self.str_template)
|
|
elif isinstance(self.exp, Group):
|
|
exp = self.exp.exp
|
|
return template % exp
|
|
|
|
str_template = """
|
|
[
|
|
%s
|
|
]
|
|
"""
|
|
|
|
def _nullable(self):
|
|
return True
|
|
|
|
|
|
class Cut(Model):
|
|
def parse(self, ctx):
|
|
ctx._cut()
|
|
|
|
def _first(self, k, f):
|
|
return {('~',)}
|
|
|
|
def _to_str(self, lean=False):
|
|
return '~'
|
|
|
|
def _nullable(self):
|
|
return True
|
|
|
|
|
|
class Named(Decorator):
|
|
def __init__(self, ast=None, **kwargs):
|
|
super().__init__(ast.exp)
|
|
self.name = ast.name
|
|
|
|
def parse(self, ctx):
|
|
value = self.exp.parse(ctx)
|
|
ctx.ast[self.name] = value
|
|
return value
|
|
|
|
def defines(self):
|
|
return [(self.name, False), *super().defines()]
|
|
|
|
def _to_str(self, lean=False):
|
|
if lean:
|
|
return self.exp._to_str(lean=True)
|
|
return f'{self.name}:{self.exp._to_str(lean=lean)}'
|
|
|
|
|
|
class NamedList(Named):
|
|
def parse(self, ctx):
|
|
value = self.exp.parse(ctx)
|
|
ctx.ast._setlist(self.name, value)
|
|
return value
|
|
|
|
def defines(self):
|
|
return [(self.name, True), *super().defines()]
|
|
|
|
def _to_str(self, lean=False):
|
|
if lean:
|
|
return self.exp._to_str(lean=True)
|
|
return f'{self.name}+:{self.exp._to_str(lean=lean)!s}'
|
|
|
|
|
|
class Override(Named):
|
|
def __init__(self, ast=None, **kwargs):
|
|
super().__init__(ast=AST(name='@', exp=ast))
|
|
|
|
def defines(self):
|
|
return []
|
|
|
|
|
|
class OverrideList(NamedList):
|
|
def __init__(self, ast=None, **kwargs):
|
|
super().__init__(ast=AST(name='@', exp=ast))
|
|
|
|
def defines(self):
|
|
return []
|
|
|
|
|
|
class Special(Model):
|
|
def _first(self, k, f):
|
|
return {(self.value,)}
|
|
|
|
def _to_str(self, lean=False):
|
|
return f'?{self.value}?'
|
|
|
|
def _nullable(self):
|
|
return True
|
|
|
|
|
|
class RuleRef(Model):
|
|
def __post_init__(self):
|
|
super().__post_init__()
|
|
self.name = self.ast
|
|
|
|
def parse(self, ctx):
|
|
try:
|
|
rule = ctx._find_rule(self.name)
|
|
except KeyError:
|
|
ctx._error(self.name, exclass=FailedRef)
|
|
else:
|
|
return rule()
|
|
|
|
def missing_rules(self, rules):
|
|
if self.name not in rules:
|
|
return oset({self.name})
|
|
return oset()
|
|
|
|
def _used_rule_names(self):
|
|
return {self.name}
|
|
|
|
def _first(self, k, f):
|
|
self._firstset = oset(f[self.name]) | {ref(self.name)}
|
|
return self._firstset
|
|
|
|
def _follow(self, k, fl, a):
|
|
fl[self.name] |= a
|
|
return oset(a) | {self.name}
|
|
|
|
def firstset(self, k=1):
|
|
if self._firstset is None:
|
|
self._firstset = {ref(self.name)}
|
|
return self._firstset
|
|
|
|
def _to_str(self, lean=False):
|
|
return self.name
|
|
|
|
def is_nullable(self, ctx=None):
|
|
return ctx[self.name].is_nullable(ctx)
|
|
|
|
|
|
class RuleInclude(Decorator):
|
|
def __init__(self, rule):
|
|
assert isinstance(rule, Rule), str(rule.name)
|
|
super().__init__(rule.exp)
|
|
self.rule = rule
|
|
|
|
def _to_str(self, lean=False):
|
|
return f'>{self.rule.name}'
|
|
|
|
|
|
class Rule(Decorator):
|
|
def __init__(self, ast, name, exp, params, kwparams, decorators=None):
|
|
assert kwparams is None or isinstance(kwparams, Mapping), kwparams
|
|
super().__init__(exp=exp, ast=ast)
|
|
self.name = name
|
|
self.params = params
|
|
self.kwparams = kwparams
|
|
self.decorators = decorators or []
|
|
|
|
self.is_name = 'name' in self.decorators
|
|
self.base = None
|
|
self.is_leftrec = False # Starts a left recursive cycle
|
|
self.is_memoizable = 'nomemo' not in self.decorators
|
|
|
|
def parse(self, ctx):
|
|
return self._parse_rhs(ctx, self.exp)
|
|
|
|
def _parse_rhs(self, ctx, exp):
|
|
ruleinfo = RuleInfo(
|
|
self.name,
|
|
exp.parse,
|
|
self.is_leftrec,
|
|
self.is_memoizable,
|
|
self.is_name,
|
|
self.params,
|
|
self.kwparams,
|
|
)
|
|
return ctx._call(ruleinfo)
|
|
|
|
# def firstset(self, k=1):
|
|
# return self.exp.firstset(k=k)
|
|
|
|
def _first(self, k, f):
|
|
self._firstset = self.exp._first(k, f) | f[self.name]
|
|
return self._firstset
|
|
|
|
def _follow(self, k, fl, a):
|
|
return self.exp._follow(k, fl, fl[self.name])
|
|
|
|
def _nullable(self):
|
|
return Nullable.of(self.exp)
|
|
|
|
@staticmethod
|
|
def param_repr(p):
|
|
if isinstance(p, int | float) or (isinstance(p, str) and p.isalnum()):
|
|
return str(p)
|
|
else:
|
|
return repr(p)
|
|
|
|
def _to_str(self, lean=False):
|
|
comments = self.comments_str()
|
|
if lean:
|
|
params = ''
|
|
else:
|
|
params = (
|
|
', '.join(self.param_repr(p) for p in self.params)
|
|
if self.params
|
|
else ''
|
|
)
|
|
|
|
kwparams = ''
|
|
if self.kwparams:
|
|
kwparams = ', '.join(
|
|
f'{k}={self.param_repr(v)}'
|
|
for (k, v) in self.kwparams.items()
|
|
)
|
|
|
|
if params and kwparams:
|
|
params = f'({params}, {kwparams})'
|
|
elif kwparams:
|
|
params = f'({kwparams})'
|
|
elif params:
|
|
params = (
|
|
f'::{params}'
|
|
if len(self.params) == 1
|
|
else f'({params})'
|
|
)
|
|
|
|
base = f' < {self.base.name!s}' if self.base else ''
|
|
|
|
return trim(self.str_template).format(
|
|
name=self.name,
|
|
base=base,
|
|
params=params,
|
|
exp=indent(self.exp._to_str(lean=lean)),
|
|
comments=comments,
|
|
is_name='@name\n' if self.is_name else '',
|
|
)
|
|
|
|
str_template = """\
|
|
{is_name}{comments}{name}{base}{params}
|
|
=
|
|
{exp}
|
|
;
|
|
"""
|
|
|
|
|
|
class BasedRule(Rule):
|
|
def __init__(
|
|
self, ast, name, exp, base, params, kwparams, decorators=None,
|
|
):
|
|
super().__init__(
|
|
ast,
|
|
name,
|
|
exp,
|
|
params or base.params,
|
|
kwparams or base.kwparams,
|
|
decorators=decorators,
|
|
)
|
|
self.base = base
|
|
ast = AST(sequence=[self.base.exp, self.exp])
|
|
ast.set_parseinfo(self.base.parseinfo)
|
|
self.rhs = Sequence(ast=ast)
|
|
|
|
def parse(self, ctx):
|
|
return self._parse_rhs(ctx, self.rhs)
|
|
|
|
def defines(self):
|
|
return self.rhs.defines()
|
|
|
|
|
|
class Grammar(Model):
|
|
def __init__(
|
|
self,
|
|
name,
|
|
rules,
|
|
/,
|
|
config: ParserConfig | None = None,
|
|
directives: dict | None = None,
|
|
**settings,
|
|
):
|
|
super().__init__()
|
|
assert isinstance(rules, list), str(rules)
|
|
directives = directives or {}
|
|
self.directives = directives
|
|
|
|
config = ParserConfig.new(config=config, owner=self, **directives)
|
|
config = config.replace(**settings)
|
|
self.config = config
|
|
|
|
self.rules = rules
|
|
self.rulemap = {rule.name: rule for rule in rules}
|
|
|
|
config = config.merge(**directives)
|
|
|
|
if name is None:
|
|
name = self.directives.get('grammar')
|
|
if name is None:
|
|
name = Path(config.filename).stem
|
|
self.name = name
|
|
|
|
missing = self.missing_rules(oset(r.name for r in self.rules))
|
|
if missing:
|
|
msg = '\n'.join(['', *sorted(missing)])
|
|
raise GrammarError('Unknown rules, no parser generated:' + msg)
|
|
|
|
self._calc_lookahead_sets()
|
|
if config.left_recursion:
|
|
find_left_recursion(self)
|
|
|
|
@property
|
|
def keywords(self):
|
|
return self.config.keywords
|
|
|
|
@property
|
|
def semantics(self):
|
|
return self.config.semantics
|
|
|
|
@semantics.setter
|
|
def semantics(self, value):
|
|
self.config.semantics = value
|
|
|
|
def missing_rules(self, rules):
|
|
return oset().union(
|
|
*[rule.missing_rules(rules) for rule in self.rules],
|
|
)
|
|
|
|
def _used_rule_names(self):
|
|
if not self.rules:
|
|
return {}
|
|
|
|
used = {'start', self.rules[0].name}
|
|
prev = {}
|
|
while used != prev:
|
|
prev = used
|
|
used |= oset().union(
|
|
*[
|
|
rule._used_rule_names()
|
|
for rule in self.rules
|
|
if rule.name in used
|
|
],
|
|
)
|
|
return used
|
|
|
|
def used_rules(self):
|
|
used = self._used_rule_names()
|
|
return [rule for rule in self.rules if rule.name in used]
|
|
|
|
@property
|
|
def first_sets(self):
|
|
return self._firstset
|
|
|
|
def _calc_lookahead_sets(self, k=1):
|
|
self._calc_first_sets()
|
|
self._calc_follow_sets()
|
|
|
|
def _calc_first_sets(self, k=1):
|
|
f = defaultdict(oset)
|
|
f1 = None
|
|
while f1 != f:
|
|
f1 = copy(f)
|
|
for rule in self.rules:
|
|
f[rule.name] |= rule._first(k, f)
|
|
|
|
# cache results
|
|
for rule in self.rules:
|
|
rule._firstset = f[rule.name]
|
|
|
|
def _calc_follow_sets(self, k=1):
|
|
fl = defaultdict(oset)
|
|
fl1 = None
|
|
while fl1 != fl:
|
|
fl1 = copy(fl)
|
|
for rule in self.rules:
|
|
rule._follow(k, fl, oset())
|
|
|
|
for rule in self.rules:
|
|
rule._follow_set = fl[rule.name]
|
|
|
|
def parse(self, text: str, /, config: ParserConfig | None = None, ctx=None, **settings): # type: ignore[override]
|
|
config = self.config.replace_config(config)
|
|
config = config.replace(**settings)
|
|
|
|
start = config.effective_rule_name()
|
|
if start is None:
|
|
start = self.rules[0].name
|
|
config.start_rule = start # FIXME
|
|
|
|
if ctx is None:
|
|
ctx = ModelContext(self.rules, config=config)
|
|
return ctx.parse(text, config=config)
|
|
|
|
def nodecount(self):
|
|
return 1 + sum(r.nodecount() for r in self.rules)
|
|
|
|
def _to_str(self, lean=False):
|
|
regex_directives = {'comments', 'eol_comments', 'whitespace'}
|
|
ustr_directives = {'comments', 'grammar'}
|
|
string_directives = {'namechars'}
|
|
|
|
directives = ''
|
|
for directive, value in self.directives.items():
|
|
fmt = dict( # noqa: C408
|
|
name=directive,
|
|
frame='/' if directive in regex_directives else '',
|
|
value=(
|
|
repr(value)
|
|
if directive in string_directives
|
|
else str(value)
|
|
if directive in ustr_directives
|
|
else value
|
|
),
|
|
)
|
|
directives += '@@{name} :: {frame}{value}{frame}\n'.format(**fmt)
|
|
if directives:
|
|
directives += '\n'
|
|
|
|
keywords = '\n'.join(
|
|
'@@keyword :: ' + ' '.join(repr(k) for k in c if k is not None)
|
|
for c in chunks(sorted(self.keywords), 8)
|
|
).strip()
|
|
keywords = '\n\n' + keywords + '\n' if keywords else ''
|
|
|
|
rules = (
|
|
'\n\n'.join(str(rule._to_str(lean=lean)) for rule in self.rules)
|
|
).rstrip() + '\n'
|
|
return directives + keywords + rules
|