← Files AgentBrokoARCHIVED FILE
.venv/Lib/site-packages/parsimonious/tests/test_grammar.py
25.4 KB · Sep 30, 2026 · 23:15 UTC
# coding=utf-8
from sys import version_info
from unittest import TestCase
import pytest
from parsimonious.exceptions import BadGrammar, LeftRecursionError, ParseError, UndefinedLabel, VisitationError
from parsimonious.expressions import Literal, Lookahead, Regex, Sequence, TokenMatcher, is_callable
from parsimonious.grammar import rule_grammar, rule_syntax, RuleVisitor, Grammar, TokenGrammar, LazyReference
from parsimonious.nodes import Node
from parsimonious.utils import Token
class BootstrappingGrammarTests(TestCase):
"""Tests for the expressions in the grammar that parses the grammar
definition syntax"""
def test_quantifier(self):
text = '*'
quantifier = rule_grammar['quantifier']
self.assertEqual(quantifier.parse(text),
Node(quantifier, text, 0, 1, children=[
Node(quantifier.members[0], text, 0, 1), Node(rule_grammar['_'], text, 1, 1)]))
text = '?'
self.assertEqual(quantifier.parse(text),
Node(quantifier, text, 0, 1, children=[
Node(quantifier.members[0], text, 0, 1), Node(rule_grammar['_'], text, 1, 1)]))
text = '+'
self.assertEqual(quantifier.parse(text),
Node(quantifier, text, 0, 1, children=[
Node(quantifier.members[0], text, 0, 1), Node(rule_grammar['_'], text, 1, 1)]))
def test_spaceless_literal(self):
text = '"anything but quotes#$*&^"'
spaceless_literal = rule_grammar['spaceless_literal']
self.assertEqual(spaceless_literal.parse(text),
Node(spaceless_literal, text, 0, len(text), children=[
Node(spaceless_literal.members[0], text, 0, len(text))]))
text = r'''r"\""'''
self.assertEqual(spaceless_literal.parse(text),
Node(spaceless_literal, text, 0, 5, children=[
Node(spaceless_literal.members[0], text, 0, 5)]))
def test_regex(self):
text = '~"[a-zA-Z_][a-zA-Z_0-9]*"LI'
regex = rule_grammar['regex']
self.assertEqual(rule_grammar['regex'].parse(text),
Node(regex, text, 0, len(text), children=[
Node(Literal('~'), text, 0, 1),
Node(rule_grammar['spaceless_literal'], text, 1, 25, children=[
Node(rule_grammar['spaceless_literal'].members[0], text, 1, 25)]),
Node(regex.members[2], text, 25, 27),
Node(rule_grammar['_'], text, 27, 27)]))
def test_successes(self):
"""Make sure the PEG recognition grammar succeeds on various inputs."""
self.assertTrue(rule_grammar['label'].parse('_'))
self.assertTrue(rule_grammar['label'].parse('jeff'))
self.assertTrue(rule_grammar['label'].parse('_THIS_THING'))
self.assertTrue(rule_grammar['atom'].parse('some_label'))
self.assertTrue(rule_grammar['atom'].parse('"some literal"'))
self.assertTrue(rule_grammar['atom'].parse('~"some regex"i'))
self.assertTrue(rule_grammar['quantified'].parse('~"some regex"i*'))
self.assertTrue(rule_grammar['quantified'].parse('thing+'))
self.assertTrue(rule_grammar['quantified'].parse('"hi"?'))
self.assertTrue(rule_grammar['term'].parse('this'))
self.assertTrue(rule_grammar['term'].parse('that+'))
self.assertTrue(rule_grammar['sequence'].parse('this that? other'))
self.assertTrue(rule_grammar['ored'].parse('this / that+ / "other"'))
# + is higher precedence than &, so 'anded' should match the whole
# thing:
self.assertTrue(rule_grammar['lookahead_term'].parse('&this+'))
self.assertTrue(rule_grammar['expression'].parse('this'))
self.assertTrue(rule_grammar['expression'].parse('this? that other*'))
self.assertTrue(rule_grammar['expression'].parse('&this / that+ / "other"'))
self.assertTrue(rule_grammar['expression'].parse('this / that? / "other"+'))
self.assertTrue(rule_grammar['expression'].parse('this? that other*'))
self.assertTrue(rule_grammar['rule'].parse('this = that\r'))
self.assertTrue(rule_grammar['rule'].parse('this = the? that other* \t\r'))
self.assertTrue(rule_grammar['rule'].parse('the=~"hi*"\n'))
self.assertTrue(rule_grammar.parse('''
this = the? that other*
that = "thing"
the=~"hi*"
other = "ahoy hoy"
'''))
class RuleVisitorTests(TestCase):
"""Tests for ``RuleVisitor``
As I write these, Grammar is not yet fully implemented. Normally, there'd
be no reason to use ``RuleVisitor`` directly.
"""
def test_round_trip(self):
"""Test a simple round trip.
Parse a simple grammar, turn the parse tree into a map of expressions,
and use that to parse another piece of text.
Not everything was implemented yet, but it was a big milestone and a
proof of concept.
"""
tree = rule_grammar.parse('''number = ~"[0-9]+"\n''')
rules, default_rule = RuleVisitor().visit(tree)
text = '98'
self.assertEqual(default_rule.parse(text), Node(default_rule, text, 0, 2))
def test_undefined_rule(self):
"""Make sure we throw the right exception on undefined rules."""
tree = rule_grammar.parse('boy = howdy\n')
self.assertRaises(UndefinedLabel, RuleVisitor().visit, tree)
def test_optional(self):
tree = rule_grammar.parse('boy = "howdy"?\n')
rules, default_rule = RuleVisitor().visit(tree)
howdy = 'howdy'
# It should turn into a Node from the Optional and another from the
# Literal within.
self.assertEqual(default_rule.parse(howdy), Node(default_rule, howdy, 0, 5, children=[
Node(Literal("howdy"), howdy, 0, 5)]))
def function_rule(text, pos):
"""This is an example of a grammar rule implemented as a function, and is
provided as a test fixture."""
token = 'function'
return pos + len(token) if text[pos:].startswith(token) else None
class GrammarTests(TestCase):
"""Integration-test ``Grammar``: feed it a PEG and see if it works."""
def method_rule(self, text, pos):
"""This is an example of a grammar rule implemented as a method, and is
provided as a test fixture."""
token = 'method'
return pos + len(token) if text[pos:].startswith(token) else None
@staticmethod
def descriptor_rule(text, pos):
"""This is an example of a grammar rule implemented as a descriptor,
and is provided as a test fixture."""
token = 'descriptor'
return pos + len(token) if text[pos:].startswith(token) else None
rules = {"descriptor_rule": descriptor_rule}
def test_expressions_from_rules(self):
"""Test the ``Grammar`` base class's ability to compile an expression
tree from rules.
That the correct ``Expression`` tree is built is already tested in
``RuleGrammarTests``. This tests only that the ``Grammar`` base class's
``_expressions_from_rules`` works.
"""
greeting_grammar = Grammar('greeting = "hi" / "howdy"')
tree = greeting_grammar.parse('hi')
self.assertEqual(tree, Node(greeting_grammar['greeting'], 'hi', 0, 2, children=[
Node(Literal('hi'), 'hi', 0, 2)]))
def test_unicode(self):
"""Assert that a ``Grammar`` can convert into a string-formatted series
of rules."""
grammar = Grammar(r"""
bold_text = bold_open text bold_close
text = ~"[A-Z 0-9]*"i
bold_open = "(("
bold_close = "))"
""")
lines = str(grammar).splitlines()
self.assertEqual(lines[0], 'bold_text = bold_open text bold_close')
self.assertTrue("text = ~'[A-Z 0-9]*'i%s" % ('u' if version_info >= (3,) else '')
in lines)
self.assertTrue("bold_open = '(('" in lines)
self.assertTrue("bold_close = '))'" in lines)
self.assertEqual(len(lines), 4)
def test_match(self):
"""Make sure partial-matching (with pos) works."""
grammar = Grammar(r"""
bold_text = bold_open text bold_close
text = ~"[A-Z 0-9]*"i
bold_open = "(("
bold_close = "))"
""")
s = ' ((boo))yah'
self.assertEqual(grammar.match(s, pos=1), Node(grammar['bold_text'], s, 1, 8, children=[
Node(grammar['bold_open'], s, 1, 3),
Node(grammar['text'], s, 3, 6),
Node(grammar['bold_close'], s, 6, 8)]))
def test_bad_grammar(self):
"""Constructing a Grammar with bad rules should raise ParseError."""
self.assertRaises(ParseError, Grammar, 'just a bunch of junk')
def test_comments(self):
"""Test tolerance of comments and blank lines in and around rules."""
grammar = Grammar(r"""# This is a grammar.
# It sure is.
bold_text = stars text stars # nice
text = ~"[A-Z 0-9]*"i #dude
stars = "**"
# Pretty good
#Oh yeah.#""") # Make sure a comment doesn't need a
# \n or \r to end.
self.assertEqual(list(sorted(str(grammar).splitlines())),
['''bold_text = stars text stars''',
# TODO: Unicode flag is on by default in Python 3. I wonder if we
# should turn it on all the time in Parsimonious.
"""stars = '**'""",
'''text = ~'[A-Z 0-9]*'i%s''' % ('u' if version_info >= (3,)
else '')])
def test_multi_line(self):
"""Make sure we tolerate all sorts of crazy line breaks and comments in
the middle of rules."""
grammar = Grammar("""
bold_text = bold_open # commenty comment
text # more comment
bold_close
text = ~"[A-Z 0-9]*"i
bold_open = "((" bold_close = "))"
""")
self.assertTrue(grammar.parse('((booyah))') is not None)
def test_not(self):
"""Make sure "not" predicates get parsed and work properly."""
grammar = Grammar(r'''not_arp = !"arp" ~"[a-z]+"''')
self.assertRaises(ParseError, grammar.parse, 'arp')
self.assertTrue(grammar.parse('argle') is not None)
def test_lookahead(self):
grammar = Grammar(r'''starts_with_a = &"a" ~"[a-z]+"''')
self.assertRaises(ParseError, grammar.parse, 'burp')
s = 'arp'
self.assertEqual(grammar.parse('arp'), Node(grammar['starts_with_a'], s, 0, 3, children=[
Node(Lookahead(Literal('a')), s, 0, 0),
Node(Regex(r'[a-z]+'), s, 0, 3)]))
def test_parens(self):
grammar = Grammar(r'''sequence = "chitty" (" " "bang")+''')
# Make sure it's not as if the parens aren't there:
self.assertRaises(ParseError, grammar.parse, 'chitty bangbang')
s = 'chitty bang bang'
self.assertEqual(str(grammar.parse(s)),
"""<Node called "sequence" matching "chitty bang bang">
<Node matching "chitty">
<Node matching " bang bang">
<Node matching " bang">
<Node matching " ">
<Node matching "bang">
<Node matching " bang">
<Node matching " ">
<Node matching "bang">""")
def test_resolve_refs_order(self):
"""Smoke-test a circumstance where lazy references don't get resolved."""
grammar = Grammar("""
expression = "(" terms ")"
terms = term+
term = number
number = ~r"[0-9]+"
""")
grammar.parse('(34)')
def test_resolve_refs_completeness(self):
"""Smoke-test another circumstance where lazy references don't get resolved."""
grammar = Grammar(r"""
block = "{" _ item* "}" _
# An item is an element of a block.
item = number / word / block / paren
# Parens are for delimiting subexpressions.
paren = "(" _ item* ")" _
# Words are barewords, unquoted things, other than literals, that can live
# in lists. We may renege on some of these chars later, especially ".". We
# may add Unicode.
word = spaceless_word _
spaceless_word = ~r"[-a-z`~!@#$%^&*_+=|\\;<>,.?][-a-z0-9`~!@#$%^&*_+=|\\;<>,.?]*"i
number = ~r"[0-9]+" _ # There are decimals and strings and other stuff back on the "parsing" branch, once you get this working.
_ = meaninglessness*
meaninglessness = whitespace
whitespace = ~r"\s+"
""")
grammar.parse('{log (add 3 to 5)}')
def test_infinite_loop(self):
"""Smoke-test a grammar that was causing infinite loops while building.
This was going awry because the "int" rule was never getting marked as
resolved, so it would just keep trying to resolve it over and over.
"""
Grammar("""
digits = digit+
int = digits
digit = ~"[0-9]"
number = int
main = number
""")
def test_circular_toplevel_reference(self):
with pytest.raises(VisitationError):
Grammar("""
foo = bar
bar = foo
""")
with pytest.raises(VisitationError):
Grammar("""
foo = foo
bar = foo
""")
with pytest.raises(VisitationError):
Grammar("""
foo = bar
bar = baz
baz = foo
""")
def test_right_recursive(self):
"""Right-recursive refs should resolve."""
grammar = Grammar("""
digits = digit digits?
digit = ~r"[0-9]"
""")
self.assertTrue(grammar.parse('12') is not None)
def test_badly_circular(self):
"""Uselessly circular references should be detected by the grammar
compiler."""
self.skipTest('We have yet to make the grammar compiler detect these.')
Grammar("""
foo = bar
bar = foo
""")
def test_parens_with_leading_whitespace(self):
"""Make sure a parenthesized expression is allowed to have leading
whitespace when nested directly inside another."""
Grammar("""foo = ( ("c") )""").parse('c')
def test_single_quoted_literals(self):
Grammar("""foo = 'a' '"'""").parse('a"')
def test_simple_custom_rules(self):
"""Run 2-arg custom-coded rules through their paces."""
grammar = Grammar("""
bracketed_digit = start digit end
start = '['
end = ']'""",
digit=lambda text, pos:
(pos + 1) if text[pos].isdigit() else None)
s = '[6]'
self.assertEqual(grammar.parse(s),
Node(grammar['bracketed_digit'], s, 0, 3, children=[
Node(grammar['start'], s, 0, 1),
Node(grammar['digit'], s, 1, 2),
Node(grammar['end'], s, 2, 3)]))
def test_complex_custom_rules(self):
"""Run 5-arg custom rules through their paces.
Incidentally tests returning an actual Node from the custom rule.
"""
grammar = Grammar("""
bracketed_digit = start digit end
start = '['
end = ']'
real_digit = '6'""",
# In this particular implementation of the digit rule, no node is
# generated for `digit`; it falls right through to `real_digit`.
# I'm not sure if this could lead to problems; I can't think of
# any, but it's probably not a great idea.
digit=lambda text, pos, cache, error, grammar:
grammar['real_digit'].match_core(text, pos, cache, error))
s = '[6]'
self.assertEqual(grammar.parse(s),
Node(grammar['bracketed_digit'], s, 0, 3, children=[
Node(grammar['start'], s, 0, 1),
Node(grammar['real_digit'], s, 1, 2),
Node(grammar['end'], s, 2, 3)]))
def test_lazy_custom_rules(self):
"""Make sure LazyReferences manually shoved into custom rules are
resolved.
Incidentally test passing full-on Expressions as custom rules and
having a custom rule as the default one.
"""
grammar = Grammar("""
four = '4'
five = '5'""",
forty_five=Sequence(LazyReference('four'),
LazyReference('five'),
name='forty_five')).default('forty_five')
s = '45'
self.assertEqual(grammar.parse(s),
Node(grammar['forty_five'], s, 0, 2, children=[
Node(grammar['four'], s, 0, 1),
Node(grammar['five'], s, 1, 2)]))
def test_unconnected_custom_rules(self):
"""Make sure custom rules that aren't hooked to any other rules still
get included in the grammar and that lone ones get set as the
default.
Incidentally test Grammar's `rules` default arg.
"""
grammar = Grammar(one_char=lambda text, pos: pos + 1).default('one_char')
s = '4'
self.assertEqual(grammar.parse(s),
Node(grammar['one_char'], s, 0, 1))
def test_callability_of_routines(self):
self.assertTrue(is_callable(function_rule))
self.assertTrue(is_callable(self.method_rule))
self.assertTrue(is_callable(self.rules['descriptor_rule']))
def test_callability_custom_rules(self):
"""Confirms that functions, methods and method descriptors can all be
used to supply custom grammar rules.
"""
grammar = Grammar("""
default = function method descriptor
""",
function=function_rule,
method=self.method_rule,
descriptor=self.rules['descriptor_rule'],
)
result = grammar.parse('functionmethoddescriptor')
rule_names = [node.expr.name for node in result.children]
self.assertEqual(rule_names, ['function', 'method', 'descriptor'])
def test_lazy_default_rule(self):
"""Make sure we get an actual rule set as our default rule, even when
the first rule has forward references and is thus a LazyReference at
some point during grammar compilation.
"""
grammar = Grammar(r"""
styled_text = text
text = "hi"
""")
self.assertEqual(grammar.parse('hi'), Node(grammar['text'], 'hi', 0, 2))
def test_immutable_grammar(self):
"""Make sure that a Grammar is immutable after being created."""
grammar = Grammar(r"""
foo = 'bar'
""")
def mod_grammar(grammar):
grammar['foo'] = 1
self.assertRaises(TypeError, mod_grammar, [grammar])
def mod_grammar(grammar):
new_grammar = Grammar(r"""
baz = 'biff'
""")
grammar.update(new_grammar)
self.assertRaises(AttributeError, mod_grammar, [grammar])
def test_repr(self):
self.assertTrue(repr(Grammar(r'foo = "a"')))
def test_rule_ordering_is_preserved(self):
grammar = Grammar('\n'.join('r%s = "something"' % i for i in range(100)))
self.assertEqual(
list(grammar.keys()),
['r%s' % i for i in range(100)])
def test_rule_ordering_is_preserved_on_shallow_copies(self):
grammar = Grammar('\n'.join('r%s = "something"' % i for i in range(100)))._copy()
self.assertEqual(
list(grammar.keys()),
['r%s' % i for i in range(100)])
def test_repetitions(self):
grammar = Grammar(r'''
left_missing = "a"{,5}
right_missing = "a"{5,}
exact = "a"{5}
range = "a"{2,5}
optional = "a"?
plus = "a"+
star = "a"*
''')
should_parse = [
("left_missing", ["a" * i for i in range(6)]),
("right_missing", ["a" * i for i in range(5, 8)]),
("exact", ["a" * 5]),
("range", ["a" * i for i in range(2, 6)]),
("optional", ["", "a"]),
("plus", ["a", "aa"]),
("star", ["", "a", "aa"]),
]
for rule, examples in should_parse:
for example in examples:
assert grammar[rule].parse(example)
should_not_parse = [
("left_missing", ["a" * 6]),
("right_missing", ["a" * i for i in range(5)]),
("exact", ["a" * i for i in list(range(5)) + list(range(6, 10))]),
("range", ["a" * i for i in list(range(2)) + list(range(6, 10))]),
("optional", ["aa"]),
("plus", [""]),
("star", ["b"]),
]
for rule, examples in should_not_parse:
for example in examples:
with pytest.raises(ParseError):
grammar[rule].parse(example)
def test_equal(self):
grammar_def = (r"""
x = y / z / ""
y = "y" x
z = "z" x
""")
assert Grammar(grammar_def) == Grammar(grammar_def)
self.assertEqual(Grammar(rule_syntax), Grammar(rule_syntax))
self.assertNotEqual(Grammar('expr = ~"[a-z]{1,3}"'), Grammar('expr = ~"[a-z]{2,3}"'))
self.assertNotEqual(Grammar('expr = ~"[a-z]{1,3}"'), Grammar('expr = ~"[a-z]{1,4}"'))
self.assertNotEqual(Grammar('expr = &"a"'), Grammar('expr = !"a"'))
class TokenGrammarTests(TestCase):
"""Tests for the TokenGrammar class and associated machinery"""
def test_parse_success(self):
"""Token literals should work."""
s = [Token('token1'), Token('token2')]
grammar = TokenGrammar("""
foo = token1 "token2"
token1 = "token1"
""")
self.assertEqual(grammar.parse(s),
Node(grammar['foo'], s, 0, 2, children=[
Node(grammar['token1'], s, 0, 1),
Node(TokenMatcher('token2'), s, 1, 2)]))
def test_parse_failure(self):
"""Parse failures should work normally with token literals."""
grammar = TokenGrammar("""
foo = "token1" "token2"
""")
with pytest.raises(ParseError) as e:
grammar.parse([Token('tokenBOO'), Token('token2')])
assert "Rule 'foo' didn't match at" in str(e.value)
def test_token_repr(self):
t = Token('💣')
self.assertTrue(isinstance(t.__repr__(), str))
self.assertEqual('<Token "💣">', t.__repr__())
def test_token_star_plus_expressions(self):
a = Token("a")
b = Token("b")
grammar = TokenGrammar("""
foo = "a"*
bar = "a"+
""")
assert grammar["foo"].parse([]) is not None
assert grammar["foo"].parse([a]) is not None
assert grammar["foo"].parse([a, a]) is not None
with pytest.raises(ParseError):
grammar["foo"].parse([a, b])
with pytest.raises(ParseError):
grammar["foo"].parse([b])
assert grammar["bar"].parse([a]) is not None
with pytest.raises(ParseError):
grammar["bar"].parse([a, b])
with pytest.raises(ParseError):
grammar["bar"].parse([b])
def test_precedence_of_string_modifiers():
# r"strings", etc. should be parsed as a single literal, not r followed
# by a string literal.
g = Grammar(r"""
escaped_bell = r"\b"
r = "irrelevant"
""")
assert isinstance(g["escaped_bell"], Literal)
assert g["escaped_bell"].literal == "\\b"
with pytest.raises(ParseError):
g.parse("irrelevant\b")
g2 = Grammar(r"""
escaped_bell = r"\b"
""")
assert g2.parse("\\b")
def test_binary_grammar():
g = Grammar(r"""
file = header body terminator
header = b"\xFF" length b"~"
length = ~rb"\d+"
body = ~b"[^\xFF]*"
terminator = b"\xFF"
""")
length = 22
assert g.parse(b"\xff22~" + (b"a" * 22) + b"\xff") is not None
def test_inconsistent_string_types_in_grammar():
with pytest.raises(VisitationError) as e:
Grammar(r"""
foo = b"foo"
bar = "bar"
""")
assert e.value.original_class is BadGrammar
with pytest.raises(VisitationError) as e:
Grammar(r"""
foo = ~b"foo"
bar = "bar"
""")
assert e.value.original_class is BadGrammar
# The following should parse without errors because they use the same
# string types:
Grammar(r"""
foo = b"foo"
bar = b"bar"
""")
Grammar(r"""
foo = "foo"
bar = "bar"
""")
def test_left_associative():
# Regression test for https://github.com/erikrose/parsimonious/issues/209
language_grammar = r"""
expression = operator_expression / non_operator_expression
non_operator_expression = number_expression
operator_expression = expression "+" non_operator_expression
number_expression = ~"[0-9]+"
"""
grammar = Grammar(language_grammar)
with pytest.raises(LeftRecursionError) as e:
grammar["operator_expression"].parse("1+2")
assert "Parsimonious is a packrat parser, so it can't handle left recursion." in str(e.value)
SHA-256: d89371bc9eb53498dd027f488516cb3e82f86f7f35463b4fa3428f9908f013c3