kitty/gen/__main__.py
Kovid Goyal b815054414
Much faster SIMD UTF-8 decoding of valid UTF-8
Decode 64 byte blocks of valid UTF-8 containing no ESC using the
approach of simdutf: blocks are validated with the lookup tables of
Keiser and Lemire and converted twelve bytes at a time using shuffles
looked up from a mask of the bytes that end code points, with fast
paths for runs of ASCII, two byte and three byte sequences. Anything
else, ESC, invalid UTF-8 and the end of the input, is decoded as
before.

The shuffle tables are generated by gen/utf8_decode_tables.py.

This decodes the text of the unicode benchmark 2.6 times faster, faster
than simdutf's own conversion with error checking, and speeds up the
unicode benchmark by ~8%.

The new AVX2 decoder is faster than the AVX-512 one, so remove the
AVX-512 decoder and use the AVX2 one on CPUs with AVX-512 as well.

Also add a test comparing the SIMD decoders with the scalar one on long
inputs, which exercise the decoding of blocks, as the existing tests use
short inputs.
2026-11-16 13:43:39 +05:30

57 lines
1.3 KiB
Python

#!/usr/bin/env python
# License: GPLv3 Copyright: 2023, Kovid Goyal <kovid at kovidgoyal.net>
import os
import sys
def main(args: list[str] = sys.argv) -> None:
os.chdir(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
sys.path.insert(0, os.getcwd())
if len(args) == 1:
raise SystemExit('usage: python gen which')
which = args[1]
del args[1]
if which == 'apc-parsers':
from gen.apc_parsers import main
main(args)
elif which == 'config':
from gen.config import main
main(args)
elif which == 'srgb-lut':
from gen.srgb_lut import main
main(args)
elif which == 'key-constants':
from gen.key_constants import main
main(args)
elif which == 'go-code':
from gen.go_code import main
main(args)
elif which == 'wcwidth':
from gen.wcwidth import main
main(args)
elif which == 'cursors':
from gen.cursors import main
main(args)
elif which == 'utf8-decode-tables':
from gen.utf8_decode_tables import main
main(args)
elif which == 'color-names':
from gen.color_names import main
main(args)
else:
raise SystemExit(f'Unknown which: {which}')
if __name__ == '__main__':
main()