964 lines
26 KiB
Python
964 lines
26 KiB
Python
from __future__ import annotations
|
|
|
|
import ast as stdlib_ast
|
|
import functools
|
|
import sys
|
|
from contextlib import contextmanager, suppress
|
|
from copy import copy
|
|
|
|
from . import buffering, color, tokenizing
|
|
from .ast import AST
|
|
from .collections import OrderedSet as oset
|
|
from .exceptions import (
|
|
FailedCut,
|
|
FailedExpectingEndOfText,
|
|
FailedKeywordSemantics,
|
|
FailedLeftRecursion,
|
|
FailedLookahead,
|
|
FailedParse,
|
|
FailedPattern,
|
|
FailedRef,
|
|
FailedSemantics,
|
|
FailedToken,
|
|
OptionSucceeded,
|
|
ParseError,
|
|
)
|
|
from .infos import (
|
|
Alert,
|
|
MemoKey,
|
|
ParseInfo,
|
|
ParserConfig,
|
|
ParseState,
|
|
RuleInfo,
|
|
RuleResult,
|
|
)
|
|
from .tokenizing import Tokenizer
|
|
from .util import (
|
|
info,
|
|
is_list,
|
|
left_assoc,
|
|
prune_dict,
|
|
right_assoc,
|
|
safe_name,
|
|
trim,
|
|
)
|
|
from .util.unicode_characters import (
|
|
C_CUT,
|
|
C_ENTRY,
|
|
C_FAILURE,
|
|
C_RECURSION,
|
|
C_SUCCESS,
|
|
)
|
|
|
|
__all__ = ['ParseContext', 'tatsumasu', 'leftrec', 'nomemo']
|
|
|
|
|
|
# decorator for rule implementation methods
|
|
def tatsumasu(*params, **kwparams):
|
|
def decorator(impl):
|
|
@functools.wraps(impl)
|
|
def wrapper(self):
|
|
name = impl.__name__
|
|
# remove the single leading and trailing underscore
|
|
# that the parser generator added
|
|
name = name[1:-1]
|
|
is_leftrec = getattr(impl, 'is_leftrec', False)
|
|
is_memoizable = getattr(impl, 'is_memoizable', True)
|
|
is_name = getattr(impl, 'is_name', False)
|
|
ruleinfo = RuleInfo(
|
|
name,
|
|
impl,
|
|
is_leftrec,
|
|
is_memoizable,
|
|
is_name,
|
|
params,
|
|
kwparams,
|
|
)
|
|
return self._call(ruleinfo)
|
|
|
|
return wrapper
|
|
|
|
return decorator
|
|
|
|
|
|
# This is used to mark left recursive rules
|
|
def leftrec(impl):
|
|
impl.is_leftrec = True
|
|
impl.is_memoizable = False
|
|
return impl
|
|
|
|
|
|
# Marks rules for which memoization has to be turned off
|
|
# (has no effect when left recursion is turned off)
|
|
def nomemo(impl):
|
|
impl.is_memoizable = False
|
|
return impl
|
|
|
|
|
|
# Marks rules marked as @name in the grammar
|
|
def isname(impl):
|
|
impl.is_name = True
|
|
return impl
|
|
|
|
|
|
class closure(list):
|
|
def __hash__(self):
|
|
return hash(tuple(self))
|
|
|
|
|
|
class ParseContext:
|
|
def __init__(self, /, config: ParserConfig | None = None, **settings):
|
|
super().__init__()
|
|
config = ParserConfig.new(config, **settings)
|
|
self.config = config
|
|
self._active_config = self.config
|
|
|
|
self._tokenizer: Tokenizer | None = None
|
|
|
|
self._semantics = config.semantics
|
|
|
|
self._initialize_caches()
|
|
|
|
def _initialize_caches(self):
|
|
self._statestack = [ParseState()]
|
|
self._rule_stack = []
|
|
self._cut_stack = [False]
|
|
|
|
self._last_node = None
|
|
self.substate = None
|
|
self._lookahead = 0
|
|
self._furthest_exception = None
|
|
|
|
self._clear_memoization_caches()
|
|
|
|
@property
|
|
def active_config(self):
|
|
return self._active_config
|
|
|
|
@property
|
|
def semantics(self):
|
|
return self._semantics
|
|
|
|
@property
|
|
def encoding(self):
|
|
return self.active_config.encoding
|
|
|
|
@property
|
|
def parseinfo(self):
|
|
return self.active_config.parseinfo
|
|
|
|
@property
|
|
def trace(self):
|
|
return self.active_config.trace
|
|
|
|
@property
|
|
def trace_length(self):
|
|
return self.active_config.trace_length
|
|
|
|
@property
|
|
def trace_separator(self):
|
|
return self.active_config.trace_separator
|
|
|
|
@property
|
|
def trace_filename(self):
|
|
return self.active_config.trace_filename
|
|
|
|
@property
|
|
def comments_re(self):
|
|
return self.active_config.comments_re
|
|
|
|
@property
|
|
def eol_comments_re(self):
|
|
return self.active_config.eol_comments_re
|
|
|
|
@property
|
|
def whitespace(self):
|
|
return self.active_config.whitespace
|
|
|
|
@property
|
|
def ignorecase(self):
|
|
return self.active_config.ignorecase
|
|
|
|
@property
|
|
def nameguard(self):
|
|
return self.active_config.nameguard
|
|
|
|
@property
|
|
def memoize_lookaheads(self):
|
|
return self.active_config.memoize_lookaheads
|
|
|
|
@property
|
|
def left_recursion(self):
|
|
return self.active_config.left_recursion
|
|
|
|
@property
|
|
def colorize(self):
|
|
return self.active_config.colorize
|
|
|
|
@property
|
|
def keywords(self):
|
|
return self._keywords
|
|
|
|
@property
|
|
def namechars(self):
|
|
return self.active_config.namechars
|
|
|
|
def _reset(self, config: ParserConfig):
|
|
if self.active_config.colorize:
|
|
color.init()
|
|
self._initialize_caches()
|
|
self._keywords = oset(config.keywords)
|
|
self._semantics = config.semantics
|
|
if hasattr(self.semantics, 'set_context'):
|
|
self.semantics.set_context(self)
|
|
return config
|
|
|
|
def _set_furthest_exception(self, e):
|
|
if (
|
|
not self._furthest_exception
|
|
or e.pos > self._furthest_exception.pos
|
|
):
|
|
self._furthest_exception = e
|
|
|
|
def parse(self, text, /, config: ParserConfig | None = None, **settings):
|
|
config = self.config.replace_config(config)
|
|
config = config.replace(**settings)
|
|
self._active_config = config
|
|
|
|
self._reset(config)
|
|
if isinstance(text, tokenizing.Tokenizer):
|
|
tokenizer = text
|
|
elif text is not None:
|
|
cls = self.tokenizercls
|
|
tokenizer = cls(text, config=config)
|
|
else:
|
|
raise ParseError('No tokenizer or text')
|
|
self._tokenizer = tokenizer
|
|
start = self.active_config.effective_rule_name() or 'start'
|
|
|
|
try:
|
|
rule = self._find_rule(start)
|
|
return rule()
|
|
except FailedCut as e:
|
|
self._set_furthest_exception(e.nested)
|
|
raise self._furthest_exception from e
|
|
except FailedParse as e:
|
|
self._set_furthest_exception(e)
|
|
raise self._furthest_exception from e
|
|
finally:
|
|
self._clear_memoization_caches()
|
|
|
|
@property
|
|
def tokenizer(self):
|
|
return self._tokenizer
|
|
|
|
@property
|
|
def tokenizercls(self) -> type[Tokenizer]:
|
|
if self.config.tokenizercls is None:
|
|
return buffering.Buffer
|
|
else:
|
|
return self.config.tokenizercls
|
|
|
|
@property
|
|
def last_node(self):
|
|
return self._last_node
|
|
|
|
@last_node.setter
|
|
def last_node(self, value):
|
|
self._last_node = value
|
|
|
|
@property
|
|
def _pos(self):
|
|
return self._tokenizer.pos
|
|
|
|
def _clear_memoization_caches(self):
|
|
self._memos = {}
|
|
self._results = {}
|
|
self._recursion_depth = 0
|
|
|
|
def _goto(self, pos):
|
|
self._tokenizer.goto(pos)
|
|
|
|
def _next(self):
|
|
return self._tokenizer.next()
|
|
|
|
def _next_token(self, ruleinfo=None):
|
|
if ruleinfo is None or not ruleinfo.name.lstrip('_')[:1].isupper():
|
|
self._tokenizer.next_token()
|
|
|
|
def _define(self, keys, list_keys=None):
|
|
# if self.ast and isinstance(self.ast, AST):
|
|
ast = AST()
|
|
ast._define(keys, list_keys=list_keys)
|
|
ast.update(self.ast)
|
|
self.ast = ast
|
|
|
|
@property
|
|
def state(self):
|
|
return self._statestack[-1]
|
|
|
|
@property
|
|
def ast(self):
|
|
return self.state.ast
|
|
|
|
@ast.setter
|
|
def ast(self, value):
|
|
self.state.ast = value
|
|
|
|
def name_last_node(self, name):
|
|
self.ast[name] = self.last_node
|
|
|
|
def add_last_node_to_name(self, name):
|
|
self.ast._setlist(name, self.last_node)
|
|
|
|
@staticmethod
|
|
def _safe_name(name, ast):
|
|
while name in ast:
|
|
name += '_'
|
|
return name
|
|
|
|
def ast_set(self, name, value, as_list=False):
|
|
ast = self.ast
|
|
name = self._safe_name(name, ast)
|
|
|
|
previous = ast.get(name)
|
|
if previous is None:
|
|
new_value = [value] if as_list else value
|
|
elif is_list(previous):
|
|
new_value = [*previous, value]
|
|
else:
|
|
new_value = [previous, value]
|
|
|
|
ast = AST(ast, name=new_value)
|
|
self.ast = ast
|
|
|
|
def ast_append(self, name, value):
|
|
self.ast_set(name, value, as_list=True)
|
|
|
|
def _push_ast(self, copyast=False):
|
|
ast = copy(self.ast) if copyast else AST()
|
|
self.state.pos = self._pos
|
|
self._statestack.append(ParseState(ast=ast, pos=self._pos))
|
|
|
|
def _pop_ast(self):
|
|
self._statestack.pop()
|
|
self.tokenizer.goto(self.state.pos)
|
|
|
|
def _merge_ast(self):
|
|
pos = self._pos
|
|
ast = self.ast
|
|
cst = self.cst
|
|
self._statestack.pop()
|
|
self.ast = ast
|
|
self._extend_cst(cst)
|
|
self.tokenizer.goto(pos)
|
|
|
|
@property
|
|
def cst(self):
|
|
return self.state.cst
|
|
|
|
@cst.setter
|
|
def cst(self, value):
|
|
self.state.cst = value
|
|
|
|
def _push_cst(self):
|
|
self._statestack.append(ParseState(ast=self.ast))
|
|
|
|
def _pop_cst(self):
|
|
ast = self.ast
|
|
self._statestack.pop()
|
|
self.ast = ast
|
|
|
|
def _merge_cst(self, extend=True):
|
|
cst = self.cst
|
|
self._pop_cst()
|
|
if extend:
|
|
self._extend_cst(cst)
|
|
else:
|
|
self._append_cst(cst)
|
|
return cst
|
|
|
|
def _append_cst(self, node):
|
|
self.last_node = node
|
|
previous = self.cst
|
|
if previous is None:
|
|
self.cst = self._copy_node(node)
|
|
elif is_list(previous):
|
|
previous.append(node)
|
|
else:
|
|
self.cst = [previous, node]
|
|
return node
|
|
|
|
def _extend_cst(self, node):
|
|
self.last_node = node
|
|
if node is None:
|
|
return None
|
|
previous = self.cst
|
|
if previous is None:
|
|
self.cst = self._copy_node(node)
|
|
elif is_list(node):
|
|
if is_list(previous):
|
|
previous.extend(node)
|
|
else:
|
|
self.cst = [previous, *node]
|
|
elif is_list(previous):
|
|
previous.append(node)
|
|
else:
|
|
self.cst = [previous, node]
|
|
return node
|
|
|
|
def _copy_node(self, node):
|
|
if node is None:
|
|
return None
|
|
elif is_list(node):
|
|
return node[:]
|
|
else:
|
|
return node
|
|
|
|
def _is_cut_set(self):
|
|
return self._cut_stack[-1]
|
|
|
|
def _cut(self):
|
|
self._trace_cut()
|
|
self._cut_stack[-1] = True
|
|
|
|
# Kota Mizushima et al say that we can throw away
|
|
# memos for previous positions in the tokenizer under
|
|
# certain circumstances, without affecting the linearity
|
|
# of PEG parsing.
|
|
# https://kmizu.github.io/papers/paste513-mizushima.pdf
|
|
#
|
|
# We adopt the heuristic of always dropping the cache for
|
|
# positions less than the current cut position. It remains to
|
|
# be proven if doing it this way affects linearity. Empirically,
|
|
# it hasn't.
|
|
|
|
def prune(cache, cutpos):
|
|
prune_dict(cache, lambda k, _: k[0] < cutpos)
|
|
|
|
prune(self._memos, self._pos)
|
|
|
|
def _memoization(self):
|
|
return self.memoize_lookaheads or self._lookahead == 0
|
|
|
|
def _rulestack(self):
|
|
rulestack = [r.name for r in reversed(self._rule_stack)]
|
|
stack = self.trace_separator.join(rulestack)
|
|
if max(len(s) for s in stack.splitlines()) > self.trace_length:
|
|
stack = stack[: self.trace_length]
|
|
stack = stack.rsplit(self.trace_separator, 1)[0]
|
|
stack += self.trace_separator
|
|
return stack
|
|
|
|
def _find_rule(self, name):
|
|
self._error(name, exclass=FailedRef)
|
|
return lambda: None # makes static checkers happy
|
|
|
|
def _find_semantic_action(self, name):
|
|
if self.semantics is None:
|
|
return None, None
|
|
|
|
postproc = getattr(self.semantics, '_postproc', None)
|
|
|
|
action = getattr(self.semantics, safe_name(name), None)
|
|
if not callable(action):
|
|
action = getattr(self.semantics, '_default', None)
|
|
|
|
if not callable(action):
|
|
action = None
|
|
if not callable(postproc):
|
|
postproc = None
|
|
|
|
return action, postproc
|
|
|
|
def _trace(self, msg, *params, **kwargs):
|
|
if self.trace:
|
|
msg %= params
|
|
info(str(msg), file=sys.stderr)
|
|
|
|
def _trace_event(self, event):
|
|
if self.trace:
|
|
fname = ''
|
|
if self.trace_filename:
|
|
fname = self._tokenizer.line_info().filename + '\n'
|
|
|
|
lookahead = self._tokenizer.lookahead().rstrip()
|
|
if lookahead:
|
|
lookahead = '\n' + lookahead
|
|
self._trace(
|
|
'%s %s%s%s',
|
|
event + self._rulestack(),
|
|
self._tokenizer.lookahead_pos(),
|
|
color.Style.DIM + fname,
|
|
color.Style.NORMAL + lookahead + color.Style.RESET_ALL,
|
|
end='',
|
|
)
|
|
|
|
def _trace_entry(self):
|
|
self._trace_event(color.Fore.YELLOW + color.Style.BRIGHT + C_ENTRY)
|
|
|
|
def _trace_success(self):
|
|
self._trace_event(color.Fore.GREEN + color.Style.BRIGHT + C_SUCCESS)
|
|
|
|
def _trace_failure(self, ex=None):
|
|
if isinstance(ex, FailedLeftRecursion):
|
|
self._trace_recursion()
|
|
else:
|
|
self._trace_event(color.Fore.RED + color.Style.BRIGHT + C_FAILURE)
|
|
|
|
def _trace_recursion(self):
|
|
self._trace_event(color.Fore.RED + color.Style.BRIGHT + C_RECURSION)
|
|
|
|
def _trace_cut(self):
|
|
self._trace_event(color.Fore.MAGENTA + color.Style.BRIGHT + C_CUT)
|
|
|
|
def _trace_match(self, token, name=None, failed=False):
|
|
if self.trace:
|
|
fname = ''
|
|
if self.trace_filename:
|
|
fname = self._tokenizer.line_info().filename + '\n'
|
|
name = f'/{name}/' if name else ''
|
|
|
|
if not failed:
|
|
fgcolor = color.Fore.GREEN + C_SUCCESS
|
|
else:
|
|
fgcolor = color.Fore.RED + C_FAILURE
|
|
|
|
lookahead = self._tokenizer.lookahead().rstrip()
|
|
if lookahead:
|
|
lookahead = '\n' + lookahead
|
|
|
|
self._trace(
|
|
color.Style.BRIGHT + fgcolor + "'%s' %s%s%s",
|
|
token,
|
|
name,
|
|
color.Style.DIM + fname,
|
|
color.Style.NORMAL + lookahead + color.Style.RESET_ALL,
|
|
end='',
|
|
)
|
|
|
|
def _make_exception(self, item, exclass=FailedParse):
|
|
rulestack = (r.name for r in self._rule_stack[::-1])
|
|
return exclass(self.tokenizer, rulestack, item)
|
|
|
|
def _error(self, item, exclass=FailedParse):
|
|
raise self._make_exception(item, exclass=exclass)
|
|
|
|
def _fail(self):
|
|
self._error('fail')
|
|
|
|
def _get_parseinfo(self, name, pos):
|
|
endpos = self._pos
|
|
return ParseInfo(
|
|
tokenizer=self.tokenizer,
|
|
rule=name,
|
|
pos=pos,
|
|
endpos=endpos,
|
|
line=self.tokenizer.posline(pos),
|
|
endline=self.tokenizer.posline(endpos),
|
|
alerts=self.state.alerts,
|
|
)
|
|
|
|
@property
|
|
def rule(self):
|
|
return self._rule_stack[-1]
|
|
|
|
@property
|
|
def memokey(self):
|
|
return MemoKey(self._pos, self.rule, self.substate)
|
|
|
|
def _memoize(self, key, memo):
|
|
if self._memoization() and key.rule.is_memoizable:
|
|
self._memos[key] = memo
|
|
return memo
|
|
|
|
def _forget(self, key):
|
|
self._memos.pop(key, None)
|
|
|
|
def _memo_for(self, key):
|
|
return self._memos.get(key)
|
|
|
|
# if isinstance(memo, FailedLeftRecursion):
|
|
# memo = self._results.get(key, memo)
|
|
|
|
def _mkresult(self, node):
|
|
return RuleResult(node, self._pos, self.substate)
|
|
|
|
def _save_result(self, key, result):
|
|
if is_list(result.node):
|
|
result = RuleResult(
|
|
closure(result.node), result.newpos, result.newstate,
|
|
)
|
|
self._results[key] = result
|
|
|
|
def _is_recursive(self, ruleinfo):
|
|
return ruleinfo.is_leftrec
|
|
|
|
def _set_left_recursion_guard(self, key):
|
|
ex = self._make_exception(key.rule.name, exclass=FailedLeftRecursion)
|
|
self._memoize(key, ex)
|
|
|
|
def _call(self, ruleinfo):
|
|
self._rule_stack += [ruleinfo]
|
|
pos = self._pos
|
|
try:
|
|
self._trace_entry()
|
|
|
|
self._last_node = None
|
|
|
|
result = self._recursive_call(ruleinfo)
|
|
node, newpos, newstate = result
|
|
|
|
self._goto(newpos)
|
|
self.substate = newstate
|
|
self._append_cst(node)
|
|
|
|
self._trace_success()
|
|
|
|
return node
|
|
except FailedPattern:
|
|
self._error(f'Expecting <{ruleinfo.name}>')
|
|
except FailedParse as e:
|
|
self._goto(pos)
|
|
self._set_furthest_exception(e)
|
|
self._trace_failure(e)
|
|
raise
|
|
finally:
|
|
self._rule_stack.pop()
|
|
|
|
def _clear_recursion_errors(self):
|
|
def filter(key, value):
|
|
return isinstance(value, FailedLeftRecursion)
|
|
|
|
prune_dict(self._memos, filter)
|
|
|
|
def _recursive_call(self, ruleinfo):
|
|
self._next_token(ruleinfo)
|
|
key = self.memokey
|
|
|
|
if not ruleinfo.is_leftrec:
|
|
return self._invoke_rule(ruleinfo, key)
|
|
elif not self.left_recursion:
|
|
self._error('Left recursion detected', exclass=FailedLeftRecursion)
|
|
|
|
self._recursion_depth += 1
|
|
if key in self._results:
|
|
result = self._results[key]
|
|
else:
|
|
result = self._make_exception(
|
|
ruleinfo.name, exclass=FailedLeftRecursion,
|
|
)
|
|
self._results[key] = result
|
|
|
|
initial = self._pos
|
|
lastpos = initial - 1
|
|
while True:
|
|
try:
|
|
self._clear_recursion_errors()
|
|
new_result = self._invoke_rule(ruleinfo, key)
|
|
self._goto(initial)
|
|
except FailedParse:
|
|
break
|
|
|
|
if new_result.newpos > lastpos:
|
|
self._save_result(key, new_result)
|
|
lastpos = new_result.newpos
|
|
result = new_result
|
|
else:
|
|
break
|
|
self._recursion_depth -= 1
|
|
|
|
if isinstance(result, Exception):
|
|
raise result # pylint: disable=raising-non-exception
|
|
|
|
return result
|
|
|
|
def _invoke_rule(self, ruleinfo, key):
|
|
memo = self._memo_for(key)
|
|
if isinstance(memo, Exception):
|
|
raise memo
|
|
if memo:
|
|
return memo
|
|
self._set_left_recursion_guard(key)
|
|
|
|
self._push_ast()
|
|
try:
|
|
try:
|
|
self._next_token(ruleinfo)
|
|
ruleinfo.impl(self)
|
|
node = self._get_node(key.pos, ruleinfo)
|
|
node = self._invoke_semantic_rule(ruleinfo, node)
|
|
result = self._mkresult(node)
|
|
self._memoize(key, result)
|
|
return result
|
|
except FailedSemantics as e:
|
|
self._error(str(e))
|
|
except FailedParse as e:
|
|
self._memoize(key, e)
|
|
raise
|
|
finally:
|
|
self._pop_ast()
|
|
|
|
def _get_node(self, pos, ruleinfo):
|
|
node = self.ast
|
|
if not node:
|
|
node = tuple(self.cst) if is_list(self.cst) else self.cst
|
|
elif '@' in node:
|
|
node = node['@'] # override the AST
|
|
elif self.parseinfo:
|
|
node.set_parseinfo(self._get_parseinfo(ruleinfo.name, pos))
|
|
return node
|
|
|
|
def _invoke_semantic_rule(self, rule, node):
|
|
semantic_rule, postproc = self._find_semantic_action(rule.name)
|
|
if semantic_rule:
|
|
node = semantic_rule(
|
|
node, *(rule.params or ()), **(rule.kwparams or {}),
|
|
)
|
|
if callable(postproc):
|
|
postproc(self, node) # pylint: disable=not-callable
|
|
if rule.is_name:
|
|
self._check_name(node)
|
|
return node
|
|
|
|
def _token(self, token):
|
|
self._next_token()
|
|
if self.tokenizer.match(token) is None:
|
|
self._trace_match(token, failed=True)
|
|
self._error(token, exclass=FailedToken)
|
|
self._trace_match(token)
|
|
self._append_cst(token)
|
|
return token
|
|
|
|
def _constant(self, literal):
|
|
self._next_token()
|
|
self._trace_match(literal)
|
|
if isinstance(literal, str):
|
|
try:
|
|
literal = stdlib_ast.literal_eval(literal.strip())
|
|
except (ValueError, SyntaxError):
|
|
if '\n' in literal:
|
|
literal = trim(literal)
|
|
literal = eval( # noqa: S307
|
|
f'{"f" + repr(literal)}', {}, self.ast,
|
|
)
|
|
self._append_cst(literal)
|
|
return literal
|
|
|
|
def _alert(self, message, level):
|
|
self._next_token()
|
|
self._trace_match(f'{"^" * level}`{message}`', failed=True)
|
|
self.state.alerts.append(Alert(message=message, level=level))
|
|
|
|
def _pattern(self, pattern):
|
|
token = self.tokenizer.matchre(pattern)
|
|
if token is None:
|
|
self._trace_match('', pattern, failed=True)
|
|
self._error(pattern, exclass=FailedPattern)
|
|
self._trace_match(token, pattern)
|
|
self._append_cst(token)
|
|
return token
|
|
|
|
def _eof(self):
|
|
return self.tokenizer.atend()
|
|
|
|
def _eol(self):
|
|
return self.tokenizer.ateol()
|
|
|
|
def _check_eof(self):
|
|
self._next_token()
|
|
if not self.tokenizer.atend():
|
|
self._error(
|
|
'Expecting end of text', exclass=FailedExpectingEndOfText,
|
|
)
|
|
|
|
@contextmanager
|
|
def _try(self):
|
|
s = self.substate
|
|
self._push_ast(copyast=True)
|
|
self.last_node = None
|
|
try:
|
|
yield
|
|
self._merge_ast()
|
|
except FailedParse:
|
|
self._pop_ast()
|
|
self.substate = s
|
|
raise
|
|
|
|
@contextmanager
|
|
def _option(self):
|
|
self.last_node = None
|
|
self._cut_stack += [False]
|
|
try:
|
|
with self._try():
|
|
yield
|
|
raise OptionSucceeded()
|
|
except FailedCut:
|
|
raise
|
|
except FailedParse as e:
|
|
if self._is_cut_set():
|
|
raise FailedCut(e) from e
|
|
finally:
|
|
self._cut_stack.pop()
|
|
|
|
@contextmanager
|
|
def _choice(self):
|
|
self.last_node = None
|
|
with suppress(OptionSucceeded):
|
|
yield
|
|
|
|
@contextmanager
|
|
def _optional(self):
|
|
self.last_node = None
|
|
with self._choice(), self._option():
|
|
yield
|
|
|
|
@contextmanager
|
|
def _group(self):
|
|
self._push_cst()
|
|
try:
|
|
yield
|
|
self._merge_cst(extend=True)
|
|
except Exception:
|
|
self._pop_cst()
|
|
raise
|
|
|
|
@contextmanager
|
|
def _if(self):
|
|
s = self.substate
|
|
self._push_ast()
|
|
self._lookahead += 1
|
|
try:
|
|
yield
|
|
finally:
|
|
self._pop_ast() # simply discard
|
|
self._lookahead -= 1
|
|
self.substate = s
|
|
|
|
@contextmanager
|
|
def _ifnot(self):
|
|
try:
|
|
with self._if():
|
|
yield
|
|
except FailedParse:
|
|
pass
|
|
else:
|
|
self._error('', exclass=FailedLookahead)
|
|
|
|
def _isolate(self, block, drop=False):
|
|
self._push_cst()
|
|
try:
|
|
block()
|
|
cst = self.cst
|
|
finally:
|
|
self._pop_cst()
|
|
|
|
if is_list(cst):
|
|
cst = closure(cst)
|
|
if not drop:
|
|
self._append_cst(cst)
|
|
return cst
|
|
|
|
def _repeat(self, block, prefix=None, dropprefix=False):
|
|
while True:
|
|
with self._choice():
|
|
with self._option():
|
|
p = self._pos
|
|
|
|
if prefix:
|
|
self._isolate(prefix, drop=dropprefix)
|
|
self._cut()
|
|
|
|
self._isolate(block)
|
|
|
|
if self._pos == p:
|
|
self._error('empty closure')
|
|
return
|
|
|
|
def _closure(self, block, sep=None, omitsep=False):
|
|
self._push_cst()
|
|
try:
|
|
self.cst = []
|
|
with self._optional():
|
|
block()
|
|
self.cst = [self.cst]
|
|
self._repeat(block, prefix=sep, dropprefix=omitsep)
|
|
self.cst = closure(self.cst)
|
|
return self._merge_cst()
|
|
except Exception:
|
|
self._pop_cst()
|
|
raise
|
|
|
|
def _positive_closure(self, block, sep=None, omitsep=False):
|
|
self._push_cst()
|
|
try:
|
|
block()
|
|
self.cst = [self.cst]
|
|
self._repeat(block, prefix=sep, dropprefix=omitsep)
|
|
self.cst = closure(self.cst)
|
|
return self._merge_cst()
|
|
except Exception:
|
|
self._pop_cst()
|
|
raise
|
|
|
|
def _empty_closure(self):
|
|
cst = closure([])
|
|
self._append_cst(cst)
|
|
return cst
|
|
|
|
def _gather(self, block, sep):
|
|
return self._closure(block, sep=sep, omitsep=True)
|
|
|
|
def _positive_gather(self, block, sep):
|
|
return self._positive_closure(block, sep=sep, omitsep=True)
|
|
|
|
def _join(self, block, sep):
|
|
return self._closure(block, sep=sep)
|
|
|
|
def _positive_join(self, block, sep):
|
|
return self._positive_closure(block, sep=sep)
|
|
|
|
def _left_join(self, block, sep):
|
|
self.cst = left_assoc(self._positive_join(block, sep))
|
|
self.last_node = self.cst
|
|
return self.cst
|
|
|
|
def _right_join(self, block, sep):
|
|
self.cst = right_assoc(self._positive_join(block, sep))
|
|
self.last_node = self.cst
|
|
return self.cst
|
|
|
|
def _check_name(self, name):
|
|
name = str(name)
|
|
if self.ignorecase or self.tokenizer.ignorecase:
|
|
name = name.upper()
|
|
if name in self.keywords:
|
|
raise FailedKeywordSemantics(f'"{name}" is a reserved word')
|
|
|
|
def _void(self):
|
|
self.last_node = None
|
|
|
|
def _any(self):
|
|
c = self._next()
|
|
if c is None:
|
|
self._trace_match(c, failed=True)
|
|
self._error(c, exclass=FailedToken)
|
|
self._trace_match(c)
|
|
self._append_cst(c)
|
|
return c
|
|
|
|
def _skip_to(self, block):
|
|
while not self._eof():
|
|
try:
|
|
with self._if():
|
|
block()
|
|
except FailedParse:
|
|
pass
|
|
else:
|
|
break
|
|
pos = self._pos
|
|
self._next_token()
|
|
if self._pos == pos:
|
|
self._next()
|
|
block()
|