Moar speed: fast path for :;{}()[] tokens

Author: SimonSapin

tinycss/speedups.pyx

@@ -89,22 +89,28 @@ def tokenize_flat(css_source, int ignore_comments=1):
 
  tokens = []
 while pos < source_len:
-for type_ inxrange(n_tokens):
- type_name, regexp = compiled_tokens[type_]
- match = regexp(css_source, pos)
-if match:
-# First match is the longest. See comments on TOKENS above.
- css_value = match.group()
-break
+char= css_source[pos]
+ifcharin':;{}()[]':
+ type_ =-1# not parsed further anyway
+ type_name =char
+ css_value =char
 else:
-# No match.
-# "Any other character not matched by the above rules,
-# and neither a single nor a double quote."
-# ... but quotes at the start of a token are always matched
-# by STRING or BAD_STRING. So DELIM is any single character.
- type_ = DELIM
- type_name ='DELIM'
- css_value = css_source[pos]
+for type_ inxrange(n_tokens):
+ type_name, regexp = compiled_tokens[type_]
+ match = regexp(css_source, pos)
+if match:
+# First match is the longest. See comments on TOKENS above.
+ css_value = match.group()
+break
+else:
+# No match.
+# "Any other character not matched by the above rules,
+# and neither a single nor a double quote."
+# ... but quotes at the start of a token are always matched
+# by STRING or BAD_STRING. So DELIM is any single character.
+ type_ = DELIM
+ type_name ='DELIM'
+ css_value =char
  length =len(css_value)
  next_pos = pos + length
 

tinycss/token_data.py

@@ -118,29 +118,26 @@
 def_init():
 """Import-time initialization."""
 COMPILED_MACROS.clear()
-# Formatter is broken on PyPy: https://bugs.pypy.org/issue1081
-# expand_macros = functools.partial(
-# Formatter().vformat, args=(), kwargs=COMPILED_MACROS)
-
 forlineinMACROS.splitlines():
 ifline.strip():
 name, value=line.split('\t')
 COMPILED_MACROS[name.strip()] ='(?:%s)' \
 %value.format(**COMPILED_MACROS)
 
-delCOMPILED_TOKEN_REGEXPS[:]
-forlineinTOKENS.splitlines():
-ifline.strip():
-name, value=line.split('\t')
-COMPILED_TOKEN_REGEXPS.append((
-name.strip(),
-re.compile(
-value.format(**COMPILED_MACROS),
-# Case-insensitive when matching eg. uRL(foo)
-# but preserve the case in extracted groups
-re.I
- ).match
- ))
+COMPILED_TOKEN_REGEXPS[:] = (
+ (
+name.strip(),
+re.compile(
+value.format(**COMPILED_MACROS),
+# Case-insensitive when matching eg. uRL(foo)
+# but preserve the case in extracted groups
+re.I
+ ).match
+ )
+forlineinTOKENS.splitlines()
+ifline.strip()
+forname, valuein [line.split('\t')]
+ )
 
 COMPILED_TOKEN_INDEXES.clear()
 fori, (name, regexp) inenumerate(COMPILED_TOKEN_REGEXPS):

tinycss/tokenizer.py

@@ -48,20 +48,25 @@ def tokenize_flat(css_source, ignore_comments=True,
 source_len=len(css_source)
 tokens= []
 whilepos<source_len:
-fortype_, regexpincompiled_tokens:
-match=regexp(css_source, pos)
-ifmatch:
-# First match is the longest. See comments on TOKENS above.
-css_value=match.group()
-break
+char=css_source[pos]
+ifcharin':;{}()[]':
+type_=char
+css_value=char
 else:
-# No match.
-# "Any other character not matched by the above rules,
-# and neither a single nor a double quote."
-# ... but quotes at the start of a token are always matched
-# by STRING or BAD_STRING. So DELIM is any single character.
-type_='DELIM'
-css_value=css_source[pos]
+fortype_, regexpincompiled_tokens:
+match=regexp(css_source, pos)
+ifmatch:
+# First match is the longest. See comments on TOKENS above.
+css_value=match.group()
+break
+else:
+# No match.
+# "Any other character not matched by the above rules,
+# and neither a single nor a double quote."
+# ... but quotes at the start of a token are always matched
+# by STRING or BAD_STRING. So DELIM is any single character.
+type_='DELIM'
+css_value=char
 length=len(css_value)
 next_pos=pos+length