mirror of
https://github.com/python/cpython.git
synced 2026-07-29 13:16:59 -04:00
794b42ff8a
Add support for \p{property} and \P{property} escapes in Unicode (str)
regular expressions, for the properties the engine can resolve without
the unicodedata database. They are matched as CATEGORY opcodes or as
fixed sets of character ranges.
Supported in this change: many General_Category values (the groups L, N,
Z, C and the values Lu, Lt, Lm, Nd, Nl, No, Zs, Zl, Zp, Cc, Cf, Cs, Co
and Cn); the binary properties Alphabetic, Lowercase, Uppercase, Numeric,
Printable, XID_Start, XID_Continue, Cased and Case_Ignorable; the POSIX
compatibility classes; the code-point classes ASCII, Any, Assigned,
Noncharacter_Code_Point, Join_Control, Pattern_Syntax and
Pattern_White_Space; and Regional_Indicator, ASCII_Hex_Digit and
Hex_Digit.
Property and value names use loose matching (UAX #44 UAX44-LM3), so a
property may be spelled \p{Lu}, \p{gc=Lu} or \p{name=yes}.
Co-Authored-By: Claude Opus 4.8 <[email protected]>
280 lines
12 KiB
Python
280 lines
12 KiB
Python
#
|
|
# Secret Labs' Regular Expression Engine
|
|
#
|
|
# support for Unicode property escapes \p{...} and \P{...}
|
|
#
|
|
# See https://unicode.org/reports/tr18/ "Unicode Regular Expressions",
|
|
# requirement RL1.2 "Properties".
|
|
#
|
|
# The supported properties are matched either as CATEGORY opcodes, or as fixed
|
|
# sets of character ranges:
|
|
#
|
|
# * Properties emitted as CATEGORY opcodes (see _CATEGORY_PROPERTIES): \d, \s
|
|
# and \w (as digit, space and word, honouring the ASCII/LOCALE/UNICODE
|
|
# flags), the binary properties Alphabetic, Lowercase, Uppercase, Numeric,
|
|
# Printable, alnum, XID_Start, XID_Continue, Cased and Case_Ignorable, and
|
|
# the POSIX classes blank, graph, print and assigned.
|
|
#
|
|
# * General_Category values (see _GC_CATEGORY): L, Lt, Nd, Lu, N, Lm, Nl, No,
|
|
# Cf, Z, Zs, C and Cn (combinations of the simple predicates), plus Cc, Cs,
|
|
# Co, Zl and Zp as fixed ranges (see _GC_ANALYTIC).
|
|
#
|
|
# * Code-point classes given by fixed ranges (see _analytic_ranges): ASCII,
|
|
# Any, Noncharacter_Code_Point, Join_Control, Regional_Indicator, xdigit,
|
|
# ASCII_Hex_Digit, Hex_Digit, cntrl, and the immutable Pattern_Syntax and
|
|
# Pattern_White_Space.
|
|
#
|
|
|
|
from ._constants import (
|
|
IN, CATEGORY, NEGATE, RANGE, LITERAL,
|
|
CATEGORY_DIGIT, CATEGORY_NOT_DIGIT,
|
|
CATEGORY_SPACE, CATEGORY_NOT_SPACE,
|
|
CATEGORY_WORD, CATEGORY_NOT_WORD,
|
|
CATEGORY_ALPHA, CATEGORY_NOT_ALPHA,
|
|
CATEGORY_LOWER, CATEGORY_NOT_LOWER,
|
|
CATEGORY_UPPER, CATEGORY_NOT_UPPER,
|
|
CATEGORY_NUMERIC, CATEGORY_NOT_NUMERIC,
|
|
CATEGORY_PRINTABLE, CATEGORY_NOT_PRINTABLE,
|
|
CATEGORY_ALNUM, CATEGORY_NOT_ALNUM,
|
|
CATEGORY_XID_START, CATEGORY_NOT_XID_START,
|
|
CATEGORY_XID_CONTINUE, CATEGORY_NOT_XID_CONTINUE,
|
|
CATEGORY_TITLE, CATEGORY_NOT_TITLE,
|
|
CATEGORY_CASED, CATEGORY_NOT_CASED,
|
|
CATEGORY_CASE_IGNORABLE, CATEGORY_NOT_CASE_IGNORABLE,
|
|
CATEGORY_LU, CATEGORY_NOT_LU,
|
|
CATEGORY_N, CATEGORY_NOT_N,
|
|
CATEGORY_LM, CATEGORY_NOT_LM,
|
|
CATEGORY_NL, CATEGORY_NOT_NL,
|
|
CATEGORY_NO, CATEGORY_NOT_NO,
|
|
CATEGORY_CF, CATEGORY_NOT_CF,
|
|
CATEGORY_Z, CATEGORY_NOT_Z,
|
|
CATEGORY_ZS, CATEGORY_NOT_ZS,
|
|
CATEGORY_C, CATEGORY_NOT_C,
|
|
CATEGORY_CN, CATEGORY_NOT_CN,
|
|
CATEGORY_ASSIGNED, CATEGORY_NOT_ASSIGNED,
|
|
CATEGORY_BLANK, CATEGORY_NOT_BLANK,
|
|
CATEGORY_GRAPH, CATEGORY_NOT_GRAPH,
|
|
CATEGORY_PRINT, CATEGORY_NOT_PRINT,
|
|
)
|
|
|
|
MAXUNICODE = 0x10FFFF
|
|
|
|
# Properties implemented directly by the engine as (positive, negative)
|
|
# CATEGORY codes. The keys are normalised (see _normalize). digit, space and
|
|
# word reuse the \d, \s and \w categories and so are affected by the ASCII,
|
|
# LOCALE and UNICODE flags; the rest are plain Unicode properties and are not.
|
|
_CATEGORY_PROPERTIES = {
|
|
"digit": (CATEGORY_DIGIT, CATEGORY_NOT_DIGIT), # same as \d
|
|
"space": (CATEGORY_SPACE, CATEGORY_NOT_SPACE), # same as \s
|
|
# \p{White_Space} is approximated by \s (str.isspace), which also matches
|
|
# the information separators U+001C..U+001F.
|
|
"whitespace": (CATEGORY_SPACE, CATEGORY_NOT_SPACE),
|
|
"wspace": (CATEGORY_SPACE, CATEGORY_NOT_SPACE),
|
|
"word": (CATEGORY_WORD, CATEGORY_NOT_WORD), # same as \w
|
|
|
|
"alphabetic": (CATEGORY_ALPHA, CATEGORY_NOT_ALPHA),
|
|
"alpha": (CATEGORY_ALPHA, CATEGORY_NOT_ALPHA), # POSIX
|
|
"lowercase": (CATEGORY_LOWER, CATEGORY_NOT_LOWER),
|
|
"lower": (CATEGORY_LOWER, CATEGORY_NOT_LOWER), # POSIX
|
|
"uppercase": (CATEGORY_UPPER, CATEGORY_NOT_UPPER),
|
|
"upper": (CATEGORY_UPPER, CATEGORY_NOT_UPPER), # POSIX
|
|
"numeric": (CATEGORY_NUMERIC, CATEGORY_NOT_NUMERIC),
|
|
"printable": (CATEGORY_PRINTABLE, CATEGORY_NOT_PRINTABLE),
|
|
"cased": (CATEGORY_CASED, CATEGORY_NOT_CASED),
|
|
"caseignorable": (CATEGORY_CASE_IGNORABLE, CATEGORY_NOT_CASE_IGNORABLE),
|
|
# POSIX classes, the compatibility properties of UTS #18 Annex C (see the
|
|
# compound predicates in sre.c).
|
|
"blank": (CATEGORY_BLANK, CATEGORY_NOT_BLANK),
|
|
"graph": (CATEGORY_GRAPH, CATEGORY_NOT_GRAPH),
|
|
"print": (CATEGORY_PRINT, CATEGORY_NOT_PRINT),
|
|
"assigned": (CATEGORY_ASSIGNED, CATEGORY_NOT_ASSIGNED),
|
|
"alnum": (CATEGORY_ALNUM, CATEGORY_NOT_ALNUM), # POSIX
|
|
"xidstart": (CATEGORY_XID_START, CATEGORY_NOT_XID_START),
|
|
"xids": (CATEGORY_XID_START, CATEGORY_NOT_XID_START),
|
|
"xidcontinue": (CATEGORY_XID_CONTINUE, CATEGORY_NOT_XID_CONTINUE),
|
|
"xidc": (CATEGORY_XID_CONTINUE, CATEGORY_NOT_XID_CONTINUE),
|
|
}
|
|
|
|
# General_Category values matched by an engine category. CATEGORY_ALPHA
|
|
# matches exactly the L group, and CATEGORY_TITLE the Lt category;
|
|
# CATEGORY_DIGIT matches Nd (but, like \d, is restricted to ASCII under the
|
|
# ASCII flag). The gc group memberships (L = Lu|Ll|Lt|Lm|Lo, N = Nd|Nl|No)
|
|
# are given by the Unicode Standard 4.5, Table 4-4 "General_Category Values"
|
|
# (https://www.unicode.org/versions/Unicode17.0.0/core-spec/chapter-4/#G124142)
|
|
# and listed in
|
|
# https://www.unicode.org/Public/UCD/latest/ucd/PropertyValueAliases.txt
|
|
# The compound categories Lu, N, Lm, Nl, No, Cf, Z, Zs, C and Cn are
|
|
# combinations of the simple predicates (see sre.c) that reproduce the
|
|
# canonical gc partition; they are not Unicode-published identities.
|
|
_GC_CATEGORY = {
|
|
"l": (CATEGORY_ALPHA, CATEGORY_NOT_ALPHA),
|
|
"lt": (CATEGORY_TITLE, CATEGORY_NOT_TITLE),
|
|
"nd": (CATEGORY_DIGIT, CATEGORY_NOT_DIGIT),
|
|
"lu": (CATEGORY_LU, CATEGORY_NOT_LU),
|
|
"n": (CATEGORY_N, CATEGORY_NOT_N),
|
|
"lm": (CATEGORY_LM, CATEGORY_NOT_LM),
|
|
"nl": (CATEGORY_NL, CATEGORY_NOT_NL),
|
|
"no": (CATEGORY_NO, CATEGORY_NOT_NO),
|
|
"cf": (CATEGORY_CF, CATEGORY_NOT_CF),
|
|
"z": (CATEGORY_Z, CATEGORY_NOT_Z),
|
|
"zs": (CATEGORY_ZS, CATEGORY_NOT_ZS),
|
|
"c": (CATEGORY_C, CATEGORY_NOT_C),
|
|
"cn": (CATEGORY_CN, CATEGORY_NOT_CN),
|
|
}
|
|
|
|
# General_Category values whose members are fixed in every Unicode version,
|
|
# so they need no table: Cc (control, = POSIX cntrl), Cs (surrogates), Co
|
|
# (private use) and the single code points Zl and Zp. Cc, Cs and Co are the
|
|
# control codes, surrogate and private-use areas, fixed by the Unicode
|
|
# Standard 23.1, 23.6 and 23.5:
|
|
# https://www.unicode.org/versions/Unicode17.0.0/core-spec/chapter-23/
|
|
# All five are listed in
|
|
# https://www.unicode.org/Public/UCD/latest/ucd/extracted/DerivedGeneralCategory.txt
|
|
_CC_RANGES = [(0x00, 0x1F), (0x7F, 0x9F)]
|
|
_CS_RANGES = [(0xD800, 0xDFFF)]
|
|
_CO_RANGES = [(0xE000, 0xF8FF), (0xF0000, 0xFFFFD), (0x100000, 0x10FFFD)]
|
|
_GC_ANALYTIC = {
|
|
"cc": _CC_RANGES,
|
|
"cs": _CS_RANGES,
|
|
"co": _CO_RANGES,
|
|
"zl": [(0x2028, 0x2028)],
|
|
"zp": [(0x2029, 0x2029)],
|
|
}
|
|
|
|
# Pattern_Syntax and Pattern_White_Space are guaranteed immutable by the
|
|
# Unicode stability policy, so their members can be hardcoded.
|
|
# UAX #31 1.1, "Stability": https://www.unicode.org/reports/tr31/
|
|
# Members listed in https://www.unicode.org/Public/UCD/latest/ucd/PropList.txt
|
|
_PATTERN_WHITE_SPACE_RANGES = [
|
|
(0x0009, 0x000D), (0x0020, 0x0020), (0x0085, 0x0085), (0x200E, 0x200F),
|
|
(0x2028, 0x2029),
|
|
]
|
|
_PATTERN_SYNTAX_RANGES = [
|
|
(0x0021, 0x002F), (0x003A, 0x0040), (0x005B, 0x005E), (0x0060, 0x0060),
|
|
(0x007B, 0x007E), (0x00A1, 0x00A7), (0x00A9, 0x00A9), (0x00AB, 0x00AC),
|
|
(0x00AE, 0x00AE), (0x00B0, 0x00B1), (0x00B6, 0x00B6), (0x00BB, 0x00BB),
|
|
(0x00BF, 0x00BF), (0x00D7, 0x00D7), (0x00F7, 0x00F7), (0x2010, 0x2027),
|
|
(0x2030, 0x203E), (0x2041, 0x2053), (0x2055, 0x205E), (0x2190, 0x245F),
|
|
(0x2500, 0x2775), (0x2794, 0x2BFF), (0x2E00, 0x2E7F), (0x3001, 0x3003),
|
|
(0x3008, 0x3020), (0x3030, 0x3030), (0xFD3E, 0xFD3F), (0xFE45, 0xFE46),
|
|
]
|
|
|
|
# Normalised property names that introduce a General_Category value. A bare
|
|
# \p{Lu} is shorthand for \p{gc=Lu} (UTS #18 1.2.4, "Property Syntax").
|
|
_GC_KEYS = frozenset({"gc", "generalcategory"})
|
|
|
|
# Normalised value names for the truth value of a binary property; Yes/No and
|
|
# True/False are the binary value aliases of PropertyValueAliases.txt.
|
|
_TRUE_VALUES = frozenset({"yes", "y", "true", "t"})
|
|
_FALSE_VALUES = frozenset({"no", "n", "false", "f"})
|
|
|
|
|
|
def _analytic_ranges():
|
|
# Properties whose members follow directly from the code point. Keys are
|
|
# normalised.
|
|
# Noncharacter_Code_Point: U+FDD0..FDEF and the last two of every plane,
|
|
# permanently reserved (the Unicode Standard 23.7, "Noncharacters":
|
|
# https://www.unicode.org/versions/Unicode17.0.0/core-spec/chapter-23/).
|
|
noncharacter = [(0xFDD0, 0xFDEF)]
|
|
noncharacter += [(plane | 0xFFFE, plane | 0xFFFF)
|
|
for plane in range(0, MAXUNICODE + 1, 0x10000)]
|
|
# Regional_Indicator (RI): the 26 enclosed symbols A..Z, a complete fixed
|
|
# block (PropList.txt binary property).
|
|
regional_indicator = [(0x1F1E6, 0x1F1FF)]
|
|
# ASCII_Hex_Digit (= POSIX xdigit) and Hex_Digit, which adds the fullwidth
|
|
# forms. Both are complete, fixed sets (PropList.txt binary properties).
|
|
ascii_hex = [(0x30, 0x39), (0x41, 0x46), (0x61, 0x66)]
|
|
hex_digit = ascii_hex + [(0xFF10, 0xFF19), (0xFF21, 0xFF26), (0xFF41, 0xFF46)]
|
|
return {
|
|
"ascii": [(0, 0x7F)],
|
|
"any": [(0, MAXUNICODE)],
|
|
# Join_Control (U+200C ZWNJ, U+200D ZWJ; the Unicode Standard 23.2,
|
|
# "Layout Controls"), a PropList.txt binary property.
|
|
"joincontrol": [(0x200C, 0x200D)],
|
|
"regionalindicator": regional_indicator,
|
|
"ri": regional_indicator,
|
|
"noncharactercodepoint": noncharacter,
|
|
"xdigit": ascii_hex, # POSIX, ASCII only
|
|
"asciihexdigit": ascii_hex,
|
|
"ahex": ascii_hex,
|
|
"hexdigit": hex_digit,
|
|
"hex": hex_digit,
|
|
# POSIX cntrl is the General_Category Cc, a fixed set of code points.
|
|
"cntrl": _CC_RANGES,
|
|
"patternwhitespace": _PATTERN_WHITE_SPACE_RANGES,
|
|
"patws": _PATTERN_WHITE_SPACE_RANGES,
|
|
"patternsyntax": _PATTERN_SYNTAX_RANGES,
|
|
"patsyn": _PATTERN_SYNTAX_RANGES,
|
|
}
|
|
|
|
|
|
def _normalize(name):
|
|
# Unicode property and value names are matched loosely: case, spaces,
|
|
# hyphens and underscores are not significant, and an initial "is" prefix
|
|
# is ignored (UAX #44 5.9, "Matching Rules", UAX44-LM3;
|
|
# https://www.unicode.org/reports/tr44/).
|
|
name = name.lower().replace("_", "").replace("-", "").replace(" ", "")
|
|
# Strip a leading "is", unless "is" is the whole name and so not a prefix
|
|
# (e.g. the Line_Break value lb=IS).
|
|
if name != "is":
|
|
name = name.removeprefix("is")
|
|
return name
|
|
|
|
|
|
def _from_ranges(ranges, negate):
|
|
if ranges is None:
|
|
return None
|
|
items = [(LITERAL, lo) if lo == hi else (RANGE, (lo, hi))
|
|
for lo, hi in ranges]
|
|
if negate:
|
|
items.insert(0, (NEGATE, None))
|
|
return (IN, items)
|
|
|
|
|
|
def _general_category(value, negate):
|
|
# Resolve a General_Category value to a subpattern using an engine category
|
|
# or a fixed range set; unsupported values return None.
|
|
cat = _GC_CATEGORY.get(value)
|
|
if cat is not None:
|
|
return (IN, [(CATEGORY, cat[1] if negate else cat[0])])
|
|
return _from_ranges(_GC_ANALYTIC.get(value), negate)
|
|
|
|
|
|
def _truth(value):
|
|
value = _normalize(value)
|
|
if value in _TRUE_VALUES:
|
|
return True
|
|
if value in _FALSE_VALUES:
|
|
return False
|
|
return None
|
|
|
|
|
|
def parse_property(name, negate):
|
|
"""Parse the text inside \\p{...} / \\P{...}.
|
|
|
|
Return an (IN, items) subpattern, or None if the property is unknown.
|
|
"""
|
|
prop, sep, value = name.partition("=")
|
|
if sep:
|
|
key = _normalize(prop)
|
|
if key in _GC_KEYS:
|
|
return _general_category(_normalize(value), negate)
|
|
# A binary property spelled name=yes or name=no.
|
|
truth = _truth(value)
|
|
if truth is None:
|
|
return None
|
|
negate ^= not truth
|
|
cat = _CATEGORY_PROPERTIES.get(key)
|
|
if cat is not None:
|
|
return (IN, [(CATEGORY, cat[1] if negate else cat[0])])
|
|
return _from_ranges(_analytic_ranges().get(key), negate)
|
|
|
|
key = _normalize(name)
|
|
cat = _CATEGORY_PROPERTIES.get(key)
|
|
if cat is not None:
|
|
return (IN, [(CATEGORY, cat[1] if negate else cat[0])])
|
|
ranges = _analytic_ranges().get(key)
|
|
if ranges is not None:
|
|
return _from_ranges(ranges, negate)
|
|
return _general_category(key, negate)
|