add proxy for the first part and try to develop the second part
This commit is contained in:
Pierre Geeraert
@@ -0,0 +1 @@
|
||||
from ._common import * # noqa: F403
|
||||
@@ -0,0 +1,540 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import codecs
|
||||
import datetime
|
||||
import enum
|
||||
import functools
|
||||
import json
|
||||
import keyword
|
||||
import logging
|
||||
import operator
|
||||
import os
|
||||
import os.path
|
||||
import re
|
||||
import sys
|
||||
import warnings
|
||||
import weakref
|
||||
from collections.abc import Iterable, Mapping, MutableSequence
|
||||
from io import StringIO
|
||||
from itertools import zip_longest
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger('tatsu')
|
||||
logger.setLevel(logging.DEBUG)
|
||||
ch = logging.StreamHandler()
|
||||
formatter = logging.Formatter('%(message)s')
|
||||
ch.setFormatter(formatter)
|
||||
logger.addHandler(ch)
|
||||
|
||||
|
||||
ESCAPE_SEQUENCE_RE = re.compile(
|
||||
r"""
|
||||
( \\U........ # 8-digit Unicode escapes
|
||||
| \\u.... # 4-digit Unicode escapes
|
||||
| \\x.. # 2-digit Unicode escapes
|
||||
| \\[0-7]{1,3} # Octal character escapes
|
||||
| \\N\{[^}]+\} # Unicode characters by name
|
||||
| \\[\\'"abfnrtv] # Single-character escapes
|
||||
)""",
|
||||
re.UNICODE | re.VERBOSE,
|
||||
)
|
||||
|
||||
|
||||
class AsJSONMixin:
|
||||
def __json__(self, seen=None):
|
||||
return {
|
||||
'__class__': type(self).__name__,
|
||||
**asjson(self._pubdict(), seen=seen),
|
||||
}
|
||||
|
||||
def _pubdict(self):
|
||||
return {
|
||||
name: value
|
||||
for name, value in vars(self).items()
|
||||
if not name.startswith('_')
|
||||
}
|
||||
|
||||
|
||||
def program_name():
|
||||
import __main__ as main
|
||||
if package := main.__package__:
|
||||
return package
|
||||
return Path(main.__file__).name
|
||||
|
||||
|
||||
def is_posix():
|
||||
return os.name == 'posix'
|
||||
|
||||
|
||||
def _prints(*args, **kwargs):
|
||||
with StringIO() as f:
|
||||
kwargs['file'] = f
|
||||
kwargs['end'] = ''
|
||||
print(*args, **kwargs)
|
||||
return f.getvalue()
|
||||
|
||||
|
||||
def info(*args, **kwargs):
|
||||
logger.info(_prints(*args, **kwargs))
|
||||
|
||||
|
||||
def debug(*args, **kwargs):
|
||||
logger.debug(_prints(*args, **kwargs))
|
||||
|
||||
|
||||
def warning(*args, **kwargs):
|
||||
logger.warning(_prints(*args, **kwargs))
|
||||
|
||||
|
||||
def error(*args, **kwargs):
|
||||
logger.error(_prints(*args, **kwargs))
|
||||
|
||||
|
||||
def identity(*args):
|
||||
if len(args) == 1:
|
||||
return args[0]
|
||||
return args
|
||||
|
||||
|
||||
def is_list(o):
|
||||
return type(o) is list
|
||||
|
||||
|
||||
def to_list(o):
|
||||
if o is None:
|
||||
return []
|
||||
elif isinstance(o, list):
|
||||
return o
|
||||
else:
|
||||
return [o]
|
||||
|
||||
|
||||
def is_namedtuple(obj) -> bool:
|
||||
return (
|
||||
len(type(obj).__bases__) == 1
|
||||
and isinstance(obj, tuple)
|
||||
and hasattr(obj, '_asdict')
|
||||
and hasattr(obj, '_fields')
|
||||
and all(isinstance(f, str) for f in getattr(obj, '_fields', ()))
|
||||
)
|
||||
|
||||
|
||||
def simplify_list(x):
|
||||
if isinstance(x, list) and len(x) == 1:
|
||||
return simplify_list(x[0])
|
||||
return x
|
||||
|
||||
|
||||
def extend_list(x, n, default=None):
|
||||
def _null():
|
||||
pass
|
||||
|
||||
default = default or _null
|
||||
|
||||
missing = max(0, 1 + n - len(x))
|
||||
x.extend(default() for _ in range(missing))
|
||||
|
||||
|
||||
def contains_sublist(lst, sublst):
|
||||
n = len(sublst)
|
||||
return any(sublst == lst[i: i + n] for i in range(1 + len(lst) - n))
|
||||
|
||||
|
||||
def join_lists(lists):
|
||||
return functools.reduce(operator.iadd, lists, [])
|
||||
|
||||
|
||||
def flatten(o):
|
||||
if not isinstance(o, MutableSequence):
|
||||
yield o
|
||||
return
|
||||
|
||||
for item in o:
|
||||
yield from flatten(item)
|
||||
|
||||
|
||||
def compress_seq(seq):
|
||||
seen = set()
|
||||
result = []
|
||||
for x in seq:
|
||||
if x not in seen:
|
||||
result.append(x)
|
||||
seen.add(x)
|
||||
return result
|
||||
|
||||
|
||||
def eval_escapes(s):
|
||||
"""
|
||||
Given a string, evaluate escape sequences starting with backslashes as
|
||||
they would be evaluated in Python source code. For a list of these
|
||||
sequences, see: https://docs.python.org/3/reference/lexical_analysis.html
|
||||
|
||||
This is not the same as decoding the whole string with the 'unicode-escape'
|
||||
codec, because that provides no way to handle non-ASCII characters that are
|
||||
literally present in the string.
|
||||
"""
|
||||
# by Rob Speer
|
||||
|
||||
def decode_match(match):
|
||||
return codecs.decode(match.group(0), 'unicode-escape')
|
||||
|
||||
return ESCAPE_SEQUENCE_RE.sub(decode_match, s)
|
||||
|
||||
|
||||
def isiter(value):
|
||||
return isinstance(value, Iterable) and not isinstance(
|
||||
value, str | bytes | bytearray,
|
||||
)
|
||||
|
||||
|
||||
def trim(text, tabwidth=4):
|
||||
"""
|
||||
Trim text of common, leading whitespace.
|
||||
|
||||
Based on the trim algorithm of PEP 257:
|
||||
http://www.python.org/dev/peps/pep-0257/
|
||||
"""
|
||||
if not text:
|
||||
return ''
|
||||
lines = text.expandtabs(tabwidth).splitlines()
|
||||
maxindent = len(text)
|
||||
indent = maxindent
|
||||
for line in lines[1:]:
|
||||
stripped = line.lstrip()
|
||||
if stripped:
|
||||
indent = min(indent, len(line) - len(stripped))
|
||||
trimmed = [lines[0].strip()] + [
|
||||
line[indent:].rstrip() for line in lines[1:]
|
||||
]
|
||||
i = 0
|
||||
while i < len(trimmed) and not trimmed[i]:
|
||||
i += 1
|
||||
return '\n'.join(trimmed[i:])
|
||||
|
||||
|
||||
def indent(text, indent=1, multiplier=4):
|
||||
"""Indent the given block of text by indent*4 spaces"""
|
||||
if text is None:
|
||||
return ''
|
||||
text = str(text)
|
||||
if indent >= 0:
|
||||
sindent = ' ' * multiplier * indent
|
||||
text = '\n'.join((sindent + t).rstrip() for t in text.splitlines())
|
||||
return text
|
||||
|
||||
|
||||
def format_if(fmt, values):
|
||||
return fmt % values if values else ''
|
||||
|
||||
|
||||
def notnone(value, default=None):
|
||||
return value if value is not None else default
|
||||
|
||||
|
||||
def timestamp():
|
||||
return '.'.join(
|
||||
'%2.2d' % t for t in datetime.datetime.utcnow().utctimetuple()[:-2]
|
||||
)
|
||||
|
||||
|
||||
def asjson(obj, seen=None): # noqa: PLR0911, PLR0912
|
||||
if obj is None or isinstance(obj, int | float | str | bool):
|
||||
return obj
|
||||
|
||||
if seen is None:
|
||||
seen = set()
|
||||
elif id(obj) in seen:
|
||||
return f'{type(obj).__name__}@{id(obj)}'
|
||||
|
||||
if isinstance(obj, Mapping | AsJSONMixin) or isiter(obj):
|
||||
seen.add(id(obj))
|
||||
|
||||
try:
|
||||
if isinstance(obj, weakref.ReferenceType | weakref.ProxyType):
|
||||
return f'{obj.__class__.__name__}@0x{hex(id(obj)).upper()[2:]}'
|
||||
elif hasattr(obj, '__json__'):
|
||||
return obj.__json__(seen=seen)
|
||||
elif is_namedtuple(obj):
|
||||
return asjson(obj._asdict(), seen=seen)
|
||||
elif isinstance(obj, Mapping):
|
||||
result = {}
|
||||
for k, v in obj.items():
|
||||
try:
|
||||
result[k] = asjson(v, seen=seen)
|
||||
except TypeError:
|
||||
debug('Unhashable key?', type(k), str(k))
|
||||
raise
|
||||
return result
|
||||
elif isiter(obj):
|
||||
return [asjson(e, seen=seen) for e in obj]
|
||||
elif isinstance(obj, enum.Enum):
|
||||
return asjson(obj.value)
|
||||
else:
|
||||
return repr(obj)
|
||||
finally:
|
||||
# NOTE: id()s may be reused
|
||||
# https://docs.python.org/3/library/functions.html#id
|
||||
seen -= {id(obj)}
|
||||
|
||||
|
||||
def minjson(obj, typesfiltered=(str, list, dict)):
|
||||
if isinstance(obj, Mapping):
|
||||
return {
|
||||
name: minjson(value)
|
||||
for name, value in obj.items()
|
||||
if (
|
||||
not name.startswith('_')
|
||||
and value is not None
|
||||
and (value or not isinstance(value, typesfiltered))
|
||||
)
|
||||
}
|
||||
elif isiter(obj):
|
||||
return [minjson(e) for e in obj]
|
||||
else:
|
||||
return obj
|
||||
|
||||
|
||||
def plainjson(obj):
|
||||
if isinstance(obj, Mapping):
|
||||
return {
|
||||
name: plainjson(value)
|
||||
for name, value in obj.items()
|
||||
if name not in {'__class__', 'parseinfo'}
|
||||
}
|
||||
elif isinstance(obj, weakref.ReferenceType | weakref.ProxyType):
|
||||
return '@ref'
|
||||
elif isinstance(obj, str) and obj.startswith('@'):
|
||||
return '@ref'
|
||||
elif isiter(obj):
|
||||
return [plainjson(e) for e in obj]
|
||||
else:
|
||||
return obj
|
||||
|
||||
|
||||
class FallbackJSONEncoder(json.JSONEncoder):
|
||||
"""A JSON Encoder that falls back to repr() for non-JSON-aware objects."""
|
||||
|
||||
def default(self, o):
|
||||
return repr(o)
|
||||
|
||||
|
||||
def asjsons(obj):
|
||||
return json.dumps(asjson(obj), indent=2, cls=FallbackJSONEncoder)
|
||||
|
||||
|
||||
def prune_dict(d, predicate):
|
||||
"""Remove all items x where predicate(x, d[x])"""
|
||||
|
||||
keys = [k for k, v in d.items() if predicate(k, v)]
|
||||
for k in keys:
|
||||
del d[k]
|
||||
|
||||
|
||||
def safe_name(name):
|
||||
if keyword.iskeyword(name):
|
||||
return name + '_'
|
||||
return name
|
||||
|
||||
|
||||
def chunks(iterable, size, fillvalue=None):
|
||||
return zip_longest(*[iter(iterable)] * size, fillvalue=fillvalue)
|
||||
|
||||
|
||||
def generic_main(custom_main, parser_class, name='Unknown'):
|
||||
import argparse
|
||||
|
||||
class ListRules(argparse.Action):
|
||||
def __call__(self, parser, namespace, values, option_string=None):
|
||||
print('Rules:')
|
||||
for r in parser_class.rule_list():
|
||||
print(r)
|
||||
print()
|
||||
sys.exit(0)
|
||||
|
||||
argp = argparse.ArgumentParser(description=f'Simple parser for {name}.')
|
||||
addarg = argp.add_argument
|
||||
|
||||
addarg(
|
||||
'-c',
|
||||
'--color',
|
||||
help='use color in traces (requires the colorama library)',
|
||||
action='store_true',
|
||||
)
|
||||
addarg(
|
||||
'-l',
|
||||
'--list',
|
||||
action=ListRules,
|
||||
nargs=0,
|
||||
help='list all rules and exit',
|
||||
)
|
||||
addarg(
|
||||
'-n',
|
||||
'--no-nameguard',
|
||||
action='store_true',
|
||||
dest='no_nameguard',
|
||||
help="disable the 'nameguard' feature",
|
||||
)
|
||||
addarg(
|
||||
'-t', '--trace', action='store_true', help='output trace information',
|
||||
)
|
||||
addarg(
|
||||
'-w',
|
||||
'--whitespace',
|
||||
type=str,
|
||||
default=None,
|
||||
help='whitespace specification',
|
||||
)
|
||||
addarg(
|
||||
'file',
|
||||
metavar='FILE',
|
||||
help="the input file to parse or '-' for standard input",
|
||||
nargs='?',
|
||||
default='-',
|
||||
)
|
||||
addarg(
|
||||
'startrule',
|
||||
metavar='STARTRULE',
|
||||
nargs='?',
|
||||
help='the start rule for parsing',
|
||||
default=None,
|
||||
)
|
||||
|
||||
args = argp.parse_args()
|
||||
try:
|
||||
return custom_main(
|
||||
args.file,
|
||||
start=args.startrule,
|
||||
trace=args.trace,
|
||||
whitespace=args.whitespace,
|
||||
nameguard=not args.no_nameguard,
|
||||
colorize=args.color,
|
||||
)
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
|
||||
|
||||
# decorator
|
||||
def deprecated(fun):
|
||||
@functools.wraps(fun)
|
||||
def wrapper(*args, **kwargs):
|
||||
warnings.filterwarnings('default', category=DeprecationWarning)
|
||||
msg = 'Call to deprecated function {}.'
|
||||
warnings.warn(
|
||||
msg.format(fun.__name__), category=DeprecationWarning, stacklevel=2,
|
||||
)
|
||||
return fun(*args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
def left_assoc(elements):
|
||||
if not elements:
|
||||
return ()
|
||||
|
||||
it = iter(elements)
|
||||
expre = next(it)
|
||||
for e in it:
|
||||
op = e
|
||||
expre = (op, expre, next(it))
|
||||
return expre
|
||||
|
||||
|
||||
def right_assoc(elements):
|
||||
if not elements:
|
||||
return ()
|
||||
|
||||
def assoc(it):
|
||||
left = next(it)
|
||||
try:
|
||||
op = next(it)
|
||||
except StopIteration:
|
||||
return left
|
||||
else:
|
||||
return (op, left, assoc(it))
|
||||
|
||||
return assoc(iter(elements))
|
||||
|
||||
|
||||
try:
|
||||
import psutil
|
||||
except ImportError:
|
||||
|
||||
def memory_use():
|
||||
return 0
|
||||
|
||||
else:
|
||||
|
||||
def memory_use():
|
||||
process = psutil.Process(os.getpid())
|
||||
return process.memory_info().rss
|
||||
|
||||
|
||||
def try_read(filename):
|
||||
if isinstance(filename, Path):
|
||||
filename = str(filename)
|
||||
for e in ['utf-16', 'utf-8', 'latin-1', 'cp1252', 'ascii']:
|
||||
try:
|
||||
with Path(filename).open(encoding=e) as f:
|
||||
return f.read()
|
||||
except UnicodeError:
|
||||
pass
|
||||
raise UnicodeDecodeError(f"cannot find the encoding for '{filename}'")
|
||||
|
||||
|
||||
def filelist_from_patterns(patterns, ignore=None, base='.', sizesort=False):
|
||||
base = Path(base or '.').expanduser()
|
||||
|
||||
filenames = set()
|
||||
for pattern in patterns or []:
|
||||
path = base / pattern
|
||||
if path.is_file():
|
||||
filenames.add(path)
|
||||
continue
|
||||
|
||||
if path.is_dir():
|
||||
path += '/*'
|
||||
|
||||
parts = path.parts[1:] if path.is_absolute() else path.parts
|
||||
joined_pattern = str(Path().joinpath(*parts))
|
||||
filenames.update(
|
||||
p for p in Path(path.root).glob(joined_pattern) if not p.is_dir()
|
||||
)
|
||||
|
||||
filenames = list(filenames)
|
||||
|
||||
def excluded(path):
|
||||
if any(path.match(ex) for ex in ignore):
|
||||
return True
|
||||
|
||||
return any(
|
||||
any(Path(part).match(ex) for ex in ignore) for part in path.parts
|
||||
)
|
||||
|
||||
if ignore:
|
||||
filenames = [path for path in filenames if not excluded(path)]
|
||||
if sizesort:
|
||||
filenames.sort(key=lambda f: f.stat().st_size)
|
||||
|
||||
return filenames
|
||||
|
||||
|
||||
def short_relative_path(path, base='.'):
|
||||
path = Path(path)
|
||||
base = Path(base)
|
||||
common = Path(os.path.commonpath([base.resolve(), path.resolve()]))
|
||||
|
||||
if common == path.root:
|
||||
return path
|
||||
elif common == Path.home():
|
||||
up = Path('~')
|
||||
elif common == base:
|
||||
up = Path()
|
||||
else:
|
||||
n = len(base.parts) - len(common.parts)
|
||||
up = Path('../' * n)
|
||||
|
||||
rel = str(up / path.resolve().relative_to(common))
|
||||
if len(rel) < len(str(path)):
|
||||
return rel
|
||||
else:
|
||||
return str(path)
|
||||
@@ -0,0 +1,117 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Iterable
|
||||
from functools import cache
|
||||
from typing import TypeVar
|
||||
|
||||
_T = TypeVar('_T')
|
||||
|
||||
_undefined = object() # unique object for when None is not a good default
|
||||
|
||||
|
||||
def first(iterable, default=_undefined):
|
||||
"""Return the first item of *iterable*, or *default* if *iterable* is
|
||||
empty.
|
||||
|
||||
>>> first([0, 1, 2, 3])
|
||||
0
|
||||
>>> first([], 'some default')
|
||||
'some default'
|
||||
|
||||
If *default* is not provided and there are no items in the iterable,
|
||||
raise ``ValueError``.
|
||||
|
||||
:func:`first` is useful when you have a generator of expensive-to-retrieve
|
||||
values and want any arbitrary one. It is marginally shorter than
|
||||
``next(iter(iterable), default)``.
|
||||
|
||||
"""
|
||||
# NOTE: https://more-itertools.readthedocs.io/en/stable/_modules/more_itertools/more.html#first
|
||||
try:
|
||||
return next(iter(iterable))
|
||||
except StopIteration as e:
|
||||
# I'm on the edge about raising ValueError instead of StopIteration. At
|
||||
# the moment, ValueError wins, because the caller could conceivably
|
||||
# want to do something different with flow control when I raise the
|
||||
# exception, and it's weird to explicitly catch StopIteration.
|
||||
if default is _undefined:
|
||||
raise ValueError(
|
||||
'first() was called on an empty iterable, and no '
|
||||
'default value was provided.',
|
||||
) from e
|
||||
return default
|
||||
|
||||
|
||||
def match_to_find(m: re.Match):
|
||||
if m is None:
|
||||
return None
|
||||
g = m.groups(default=m.string[0:0])
|
||||
if len(g) == 1:
|
||||
return g[0]
|
||||
else:
|
||||
return g or m.group()
|
||||
|
||||
|
||||
def findalliter(pattern, string, pos=None, endpos=None, flags=0):
|
||||
"""
|
||||
like finditer(), but with return values like findall()
|
||||
|
||||
implementation taken from cpython/Modules/_sre.c/findall()
|
||||
"""
|
||||
r = (
|
||||
pattern
|
||||
if isinstance(pattern, re.Pattern)
|
||||
else re.compile(pattern, flags=flags)
|
||||
)
|
||||
if endpos is not None:
|
||||
iterator = r.finditer(string, pos=pos, endpos=endpos)
|
||||
elif pos is not None:
|
||||
iterator = r.finditer(string, pos=pos)
|
||||
else:
|
||||
iterator = r.finditer(string)
|
||||
for m in iterator:
|
||||
yield match_to_find(m)
|
||||
|
||||
|
||||
def findfirst(pattern, string, pos=None, endpos=None, flags=0, default=_undefined):
|
||||
"""
|
||||
Avoids using the inefficient findall(...)[0], or first(findall(...))
|
||||
"""
|
||||
return first(
|
||||
findalliter(pattern, string, pos=pos, endpos=endpos, flags=flags),
|
||||
default=default,
|
||||
)
|
||||
|
||||
|
||||
def topsort(nodes: Iterable[_T], order: Iterable[tuple[_T, _T]]) -> list[_T]:
|
||||
# https://en.wikipedia.org/wiki/Topological_sorting
|
||||
|
||||
order = set(order)
|
||||
result: list[_T] = [] # Empty list that will contain the sorted elements
|
||||
|
||||
def with_incoming() -> set[_T]:
|
||||
return {m for (_, m) in order}
|
||||
|
||||
# Set of all nodes with no incoming edges
|
||||
pending = list(set(nodes) - with_incoming())
|
||||
while pending:
|
||||
n = pending.pop()
|
||||
result.append(n)
|
||||
|
||||
# nodes m with an edge from n to m
|
||||
outgoing = {m for (x, m) in order if x == n}
|
||||
order -= {(n, m) for m in outgoing}
|
||||
pending.extend(outgoing - with_incoming())
|
||||
|
||||
if order:
|
||||
raise ValueError('There are cycles in the topological order')
|
||||
|
||||
return list(reversed(result)) # a topologically sorted list
|
||||
|
||||
|
||||
@cache
|
||||
def cached_re_compile(regex: re.Pattern | str | bytes) -> re.Pattern | None:
|
||||
if isinstance(regex, re.Pattern):
|
||||
return regex
|
||||
return re.compile(regex) if isinstance(regex, (str | bytes)) else None
|
||||
@@ -0,0 +1,284 @@
|
||||
import sys
|
||||
import time
|
||||
from collections import namedtuple
|
||||
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed
|
||||
from functools import partial
|
||||
from multiprocessing import Pool, cpu_count
|
||||
from pathlib import Path
|
||||
|
||||
import rich
|
||||
from rich.progress import (
|
||||
BarColumn,
|
||||
Progress,
|
||||
TaskProgressColumn,
|
||||
TextColumn,
|
||||
TimeElapsedColumn,
|
||||
TimeRemainingColumn,
|
||||
)
|
||||
|
||||
from tatsu.util.unicode_characters import (
|
||||
U_CHECK_MARK,
|
||||
U_CROSSED_SWORDS,
|
||||
)
|
||||
|
||||
from ..util import identity, memory_use, program_name, try_read
|
||||
|
||||
__all__ = ['parallel_proc', 'processing_loop']
|
||||
|
||||
|
||||
ERROR_LOG_FILENAME = 'ERROR.log'
|
||||
EOLCH = '\r' if sys.stderr.isatty() else '\n'
|
||||
sys.setrecursionlimit(2**16)
|
||||
|
||||
__Task = namedtuple('__Task', 'payload args kwargs')
|
||||
|
||||
|
||||
class ParprocResult:
|
||||
def __init__(self, payload):
|
||||
self.payload = payload
|
||||
self.outcome = None
|
||||
self.exception = None
|
||||
self.linecount = 0
|
||||
self.time = 0
|
||||
self.memory = 0
|
||||
|
||||
@property
|
||||
def success(self):
|
||||
return self.exception is None
|
||||
|
||||
def __str__(self):
|
||||
return str(self.__dict__)
|
||||
|
||||
|
||||
def process_payload(process, task, pickable=identity, reraise=False):
|
||||
start_time = time.process_time()
|
||||
result = ParprocResult(task.payload)
|
||||
try:
|
||||
outcome = process(task.payload, *task.args, **task.kwargs)
|
||||
result.memory = memory_use()
|
||||
if hasattr(outcome, 'linecount'):
|
||||
result.linecount = outcome.linecount
|
||||
else:
|
||||
result.linecount = len(try_read(task.payload).splitlines())
|
||||
result.outcome = pickable(outcome)
|
||||
except KeyboardInterrupt:
|
||||
return None
|
||||
except Exception as e:
|
||||
result.exception = e
|
||||
finally:
|
||||
result.time = time.process_time() - start_time
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _executor_pmap(executor, process, tasks):
|
||||
nworkers = max(1, cpu_count())
|
||||
n = nworkers * 8
|
||||
chunks = [tasks[i:i + n] for i in range(0, len(tasks), n)]
|
||||
for chunk in chunks:
|
||||
with executor(max_workers=nworkers) as ex:
|
||||
futures = [ex.submit(process, task) for task in chunk]
|
||||
for future in as_completed(futures):
|
||||
yield future.result()
|
||||
|
||||
|
||||
def _thread_pmap(process, tasks):
|
||||
yield from _executor_pmap(ThreadPoolExecutor, process, tasks)
|
||||
|
||||
|
||||
def _process_pmap(process, tasks):
|
||||
yield from _executor_pmap(ProcessPoolExecutor, process, tasks)
|
||||
|
||||
|
||||
def _imap_pmap(process, tasks):
|
||||
nworkers = max(1, cpu_count())
|
||||
|
||||
n = nworkers * 4
|
||||
chunks = [tasks[i:i + n] for i in range(0, len(tasks), n)]
|
||||
|
||||
count = sum(len(c) for c in chunks)
|
||||
if len(tasks) != count:
|
||||
raise RuntimeError('number of chunked tasks different %d != %d' % (len(tasks), count))
|
||||
for chunk in chunks:
|
||||
with Pool(processes=nworkers) as pool:
|
||||
yield from pool.imap_unordered(process, chunk)
|
||||
|
||||
|
||||
_pmap = _imap_pmap
|
||||
|
||||
|
||||
def parallel_proc(payloads, process, *args, **kwargs):
|
||||
pickable = kwargs.pop('pickable', identity)
|
||||
parallel = kwargs.pop('parallel', True)
|
||||
reraise = kwargs.pop('reraise', False)
|
||||
|
||||
process = partial(process_payload, process, pickable=pickable, reraise=reraise)
|
||||
tasks = [__Task(payload, args, kwargs) for payload in payloads]
|
||||
|
||||
try:
|
||||
if len(tasks) == 1:
|
||||
yield process(tasks[0])
|
||||
else:
|
||||
pmap = _pmap if parallel else map
|
||||
yield from pmap(process, tasks)
|
||||
except KeyboardInterrupt:
|
||||
return
|
||||
|
||||
|
||||
def _build_progressbar(total):
|
||||
progress = Progress(
|
||||
TextColumn(f"[progress.description]{program_name()}"),
|
||||
BarColumn(),
|
||||
# *Progress.get_default_columns(),
|
||||
TaskProgressColumn(),
|
||||
TimeElapsedColumn(),
|
||||
TimeRemainingColumn(),
|
||||
TextColumn("[progress.description]{task.description}"),
|
||||
refresh_per_second=1,
|
||||
speed_estimate_period=30.0,
|
||||
)
|
||||
task = progress.add_task('', total=total)
|
||||
return progress, task
|
||||
|
||||
|
||||
def processing_loop(filenames, process, *args, reraise=False, **kwargs): # pylint: disable=too-many-locals
|
||||
try:
|
||||
total = len(filenames)
|
||||
total_time = 0
|
||||
run_time = 0
|
||||
start_time = time.time()
|
||||
results = parallel_proc(filenames, process, *args, **kwargs)
|
||||
results = results or []
|
||||
count = 0
|
||||
success_count = 0
|
||||
success_linecount = 0
|
||||
progress, progress_task = _build_progressbar(total)
|
||||
|
||||
if total == 1:
|
||||
log = sys.stderr
|
||||
else:
|
||||
log = Path(ERROR_LOG_FILENAME).open('w')
|
||||
|
||||
with progress:
|
||||
for result in results:
|
||||
if result is None:
|
||||
continue
|
||||
count += 1
|
||||
|
||||
total_time = time.time() - start_time
|
||||
filename = Path(result.payload).name
|
||||
if result.exception:
|
||||
icon = f'[red]{U_CROSSED_SWORDS}'
|
||||
else:
|
||||
icon = f'[green]{U_CHECK_MARK}'
|
||||
|
||||
progress.update(
|
||||
progress_task,
|
||||
advance=1,
|
||||
description=f'{icon} {filename}',
|
||||
)
|
||||
|
||||
if result.exception:
|
||||
print(file=log)
|
||||
try:
|
||||
print('ERROR:', result.payload, file=log)
|
||||
print(result.exception, file=log)
|
||||
except Exception:
|
||||
# in case of errors while serializing the exception
|
||||
print('EXCEPTION', type(result.exception).__name__, file=log)
|
||||
if reraise:
|
||||
raise result.exception
|
||||
elif result.outcome is not None:
|
||||
success_count += 1
|
||||
success_linecount += result.linecount
|
||||
run_time += result.time
|
||||
yield result
|
||||
|
||||
log.flush()
|
||||
|
||||
progress.update(progress_task, advance=0, description='')
|
||||
progress.stop()
|
||||
file_process_summary(
|
||||
filenames, total_time, run_time, success_count, success_linecount, log,
|
||||
)
|
||||
except KeyboardInterrupt:
|
||||
return
|
||||
|
||||
|
||||
def file_process_progress(latest_result, count, total, total_time):
|
||||
filename = latest_result.payload
|
||||
|
||||
percent = count / total
|
||||
mb_memory = (latest_result.memory + memory_use()) // (1024 * 1024)
|
||||
|
||||
eta = (total - count) * 0.8 * total_time / (0.2 * count)
|
||||
bar = '[%-24s]' % ('#' * round(24 * percent))
|
||||
|
||||
print(
|
||||
'%3d/%-3d' % (count, total),
|
||||
bar,
|
||||
'%3d%%' % (100 * percent),
|
||||
# format_hours(total_time),
|
||||
f'{format_hours(eta)}ETA',
|
||||
format_minutes(latest_result),
|
||||
'%3dMiB' % mb_memory if mb_memory else '',
|
||||
(Path(filename).name + ' ' * 80)[:40],
|
||||
file=sys.stderr,
|
||||
end=EOLCH)
|
||||
|
||||
|
||||
def format_minutes(result):
|
||||
return f'{result.time / 60:3.0f}:{result.time % 60:04.1f}'
|
||||
|
||||
|
||||
def format_hours(time):
|
||||
return f'{time // 3600:2.0f}:{(time // 60) % 60:02.0f}:{time % 60:02.0f}'
|
||||
|
||||
|
||||
def file_process_summary(
|
||||
filenames, total_time, run_time, success_count, success_linecount, log,
|
||||
):
|
||||
filecount = 0
|
||||
linecount = 0
|
||||
for fname in filenames:
|
||||
filecount += 1
|
||||
|
||||
nlines = len(try_read(fname).splitlines())
|
||||
linecount += nlines
|
||||
|
||||
failures = filecount - success_count
|
||||
|
||||
summary_text = '''\
|
||||
-----------------------------------------------------------------------
|
||||
{:12,d} files input
|
||||
{:12,d} source lines input
|
||||
{:12,d} total lines processed
|
||||
{:12,d} successes
|
||||
{:12,d} failures
|
||||
{:12.1f}% success rate
|
||||
{:>13s} time
|
||||
{:>13s} run time
|
||||
'''
|
||||
summary_text = '\n'.join(s.strip() for s in summary_text.splitlines())
|
||||
|
||||
summary = summary_text.format(
|
||||
filecount,
|
||||
linecount,
|
||||
success_linecount,
|
||||
success_count,
|
||||
failures,
|
||||
100 * success_count / filecount if filecount != 0 else 0,
|
||||
format_hours(total_time),
|
||||
format_hours(run_time),
|
||||
)
|
||||
print(EOLCH + 80 * ' ', file=log)
|
||||
print(summary, file=log)
|
||||
print(EOLCH + 80 * ' ', file=sys.stderr)
|
||||
|
||||
if failures:
|
||||
rich.print(f'[red bold]FAILURES logged to [green]{log.name}!')
|
||||
print(file=sys.stderr)
|
||||
if log != sys.stderr:
|
||||
print(summary, file=sys.stderr)
|
||||
if failures:
|
||||
sys.exit(1)
|
||||
@@ -0,0 +1,85 @@
|
||||
import argparse
|
||||
|
||||
from . import filelist_from_patterns
|
||||
from .parproc import processing_loop
|
||||
|
||||
|
||||
def parallel_test_run(parse, options):
|
||||
def pysearch(pattern):
|
||||
if pattern.endswith('.py'):
|
||||
return pattern
|
||||
else:
|
||||
if not pattern.endswith('/'):
|
||||
pattern += '/'
|
||||
if '**' not in pattern:
|
||||
pattern += '**/'
|
||||
return pattern + '*.py'
|
||||
|
||||
try:
|
||||
patterns = [pysearch(p) for p in options.patterns]
|
||||
filenames = filelist_from_patterns(
|
||||
patterns, sizesort=options.sort, ignore=options.ignore,
|
||||
)
|
||||
|
||||
kwargs = vars(options)
|
||||
kwargs.pop('patterns', None)
|
||||
kwargs.pop('sort', None)
|
||||
kwargs.pop('ignore', None)
|
||||
parallel = not kwargs.pop('serial', False)
|
||||
|
||||
return processing_loop(parse, filenames, parallel=parallel, **kwargs)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
if options.verbose:
|
||||
raise
|
||||
|
||||
|
||||
def parse_args():
|
||||
argparser = argparse.ArgumentParser(
|
||||
prog=__package__,
|
||||
# description=DESCRIPTION,
|
||||
add_help=False,
|
||||
)
|
||||
|
||||
argparser.add_argument(
|
||||
'patterns', nargs='+', metavar='PATTERNS', help='filename patterns',
|
||||
)
|
||||
argparser.add_argument(
|
||||
'--help', '-h', help='show this help message and exit', action='help',
|
||||
)
|
||||
argparser.add_argument(
|
||||
'--ignore',
|
||||
'-i',
|
||||
metavar='PATTERN',
|
||||
help='ignore these patterns',
|
||||
action='append',
|
||||
)
|
||||
argparser.add_argument(
|
||||
'--sort', '-s', help='sort files by size', action='store_true',
|
||||
)
|
||||
argparser.add_argument(
|
||||
'--serial', '-S', help='do not run in parallel', action='store_true',
|
||||
)
|
||||
argparser.add_argument(
|
||||
'--trace',
|
||||
'-t',
|
||||
help='produce verbose output for a parse',
|
||||
action='store_true',
|
||||
)
|
||||
argparser.add_argument(
|
||||
'--verbose',
|
||||
'-v',
|
||||
help='output exceptions as they happen',
|
||||
action='store_true',
|
||||
)
|
||||
argparser.add_argument(
|
||||
'--exitfirst',
|
||||
'-x',
|
||||
help='output exceptions as they happen',
|
||||
action='store_true',
|
||||
)
|
||||
return argparser.parse_args()
|
||||
|
||||
|
||||
def generic_main(parse):
|
||||
return parallel_test_run(parse, parse_args())
|
||||
@@ -0,0 +1,54 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from tatsu.util import is_posix
|
||||
|
||||
U_LARROW = '\u2190'
|
||||
U_DLARROW = '\u2199'
|
||||
U_LDARROW = '\u21D0'
|
||||
U_UDARROW = '\u21D1'
|
||||
U_RDARROW = '\u21D2'
|
||||
U_DDARROW = '\u21D3'
|
||||
U_L_TRIPPLE_ARROW = '\u21DA'
|
||||
|
||||
U_WARNING = '\u26A0'
|
||||
|
||||
U_NOT_EQUAL_TO = '\u2260'
|
||||
U_IDENTICAL_TO = '\u2261'
|
||||
U_NOT_IDENTICAL_TO = '\u2262'
|
||||
|
||||
U_CHECK_MARK = '\u2713'
|
||||
|
||||
U_POWER_SYMBOL = '\u23FB'
|
||||
U_POWER_ON_SYMBOL = '\u23FC'
|
||||
U_POWER_OFF_SYMBOL = '\u23FD'
|
||||
|
||||
U_GREEK_SMALL_LETTER_EPSILON = '\u03B5'
|
||||
U_REGISTERED_SIGN = '\u00AE'
|
||||
U_RIENNMAN = '\u211D'
|
||||
|
||||
U_ANTICLOCKWISE_OPEN_CIRCLE_ARROW = '\u21BA'
|
||||
U_ANTICLOCKWISE_GAPPED_CIRCLE_ARROW = '\u27F2'
|
||||
|
||||
U_PUNCTUATION_SPACE = '\u2008'
|
||||
U_FOUR_PER_EM_SPACE = '\u2005'
|
||||
U_MEDIUM_MATHEMATICAL_SPACE = '\u205F'
|
||||
U_ZERO_WIDTH_NO_BREAK_SPACE = '\uFEFF'
|
||||
|
||||
U_BLACK_SCISSORS = '\u2702'
|
||||
U_CROSSED_SWORDS = '\u2694'
|
||||
|
||||
|
||||
if not is_posix():
|
||||
C_DERIVE = '<'
|
||||
C_ENTRY = '<'
|
||||
C_SUCCESS = '>'
|
||||
C_FAILURE = '!'
|
||||
C_RECURSION = 'r '
|
||||
C_CUT = '~'
|
||||
else:
|
||||
C_DERIVE = U_DLARROW
|
||||
C_ENTRY = C_DERIVE
|
||||
C_SUCCESS = U_IDENTICAL_TO
|
||||
C_FAILURE = U_NOT_IDENTICAL_TO
|
||||
C_RECURSION = U_ANTICLOCKWISE_GAPPED_CIRCLE_ARROW + U_FOUR_PER_EM_SPACE
|
||||
C_CUT = U_CROSSED_SWORDS
|
||||
Reference in New Issue
Block a user