#!/usr/bin/env python # License: GPLv3 Copyright: 2026, Kovid Goyal """ Run the kitty benchmark kitten in all installed terminals from a fixed set and present the results for comparison. Linux only. Every terminal is run with its default configuration, except for the font family, font size and window size, which are made the same for all terminals. User configuration is ignored by pointing all the XDG base directories at a temporary directory. The terminals are pinned to the fast cores on machines with heterogeneous cores and the CPU frequency governor is set to performance for the duration of the run, if possible. """ import argparse import json import os import re import shlex import shutil import signal import subprocess import sys import tempfile import time from collections.abc import Callable, Generator, Sequence from contextlib import ExitStack, contextmanager, suppress from dataclasses import dataclass, field from typing import Any BASE = os.path.dirname(os.path.abspath(__file__)) CPU_SYSFS = '/sys/devices/system/cpu' READY_FILE, GO_FILE, DONE_FILE, PROGRESS_FILE = 'ready.json', 'go', 'done.json', 'progress' ENV_PREFIXES_TO_REMOVE = ( 'KITTY_', 'TMUX', 'WEZTERM_', 'GHOSTTY_', 'ALACRITTY_', 'KONSOLE_', 'VTE_', 'TERM_PROGRAM', 'TERMINFO', 'COLORTERM', 'WINDOWID', 'XTERM_', 'PTYXIS_', 'SHELL_SESSION_ID', ) @dataclass(frozen=True) class Settings: font_family: str font_size: float columns: int lines: int window_width: int window_height: int @dataclass class Terminal: name: str executable: str # Build the command line to run cmd in the terminal, given the per-run # directory for config files and the settings build_cmd: Callable[[str, list[str], Settings], list[str]] version_cmd: list[str] = field(default_factory=list) # Run after the terminal is done, to cleanup any left over processes cleanup: Callable[[str], None] | None = None # Needs an X11 display, a private rootful Xwayland is started if DISPLAY is not set needs_x11: bool = False # Needs a D-Bus session bus and a systemd user instance (to spawn processes in scopes) needs_session: bool = False # Terminal definitions {{{ def write_file(path: str, text: str) -> str: os.makedirs(os.path.dirname(path), exist_ok=True) with open(path, 'w') as f: f.write(text) return path def fmt_size(x: float) -> str: return str(int(x)) if x == int(x) else str(x) def toml_str(x: str) -> str: return json.dumps(x) def xterm_cmd(d: str, cmd: list[str], s: Settings) -> list[str]: return ['xterm', '-fa', s.font_family, '-fs', fmt_size(s.font_size), '-geometry', f'{s.columns}x{s.lines}', '-e'] + cmd def foot_cmd(d: str, cmd: list[str], s: Settings) -> list[str]: conf = write_file(os.path.join(d, 'foot.ini'), f'font={s.font_family}:size={fmt_size(s.font_size)}\ninitial-window-size-chars={s.columns}x{s.lines}\n') return ['foot', '--config', conf] + cmd def kitty_cmd(d: str, cmd: list[str], s: Settings) -> list[str]: conf = write_file( os.path.join(d, 'kitty.conf'), '\n'.join( ( f'font_family family="{s.font_family}"', f'font_size {fmt_size(s.font_size)}', 'remember_window_size no', f'initial_window_width {s.columns}c', f'initial_window_height {s.lines}c', '', ) ), ) return [kitty_exe(), '--config', conf] + cmd def ghostty_cmd(d: str, cmd: list[str], s: Settings) -> list[str]: conf = write_file( os.path.join(d, 'ghostty.conf'), '\n'.join( ( f'font-family = {s.font_family}', f'font-size = {fmt_size(s.font_size)}', f'window-width = {s.columns}', f'window-height = {s.lines}', # Needed so that the window is created in a new process that exits when the command finishes 'gtk-single-instance = false', 'confirm-close-surface = false', 'quit-after-last-window-closed = true', '', ) ), ) return ['ghostty', '--config-default-files=false', f'--config-file={conf}', '-e'] + cmd def alacritty_config(d: str, s: Settings) -> str: return write_file( os.path.join(d, 'alacritty.toml'), '\n'.join( ( '[font]', f'size = {float(s.font_size)}', f'normal = {{ family = {toml_str(s.font_family)} }}', '[window]', f'dimensions = {{ columns = {s.columns}, lines = {s.lines} }}', '', ) ), ) def alacritty_cmd(d: str, cmd: list[str], s: Settings) -> list[str]: return ['alacritty', '--config-file', alacritty_config(d, s), '-e'] + cmd def wezterm_cmd(d: str, cmd: list[str], s: Settings) -> list[str]: conf = write_file( os.path.join(d, 'wezterm.lua'), '\n'.join( ( 'local wezterm = require "wezterm"', 'local config = wezterm.config_builder()', f'config.font = wezterm.font({json.dumps(s.font_family)})', f'config.font_size = {float(s.font_size)}', f'config.initial_cols = {s.columns}', f'config.initial_rows = {s.lines}', 'return config', '', ) ), ) return ['wezterm', '--config-file', conf, 'start', '--always-new-process', '--'] + cmd def konsole_cmd(d: str, cmd: list[str], s: Settings) -> list[str]: return [ 'konsole', '--separate', '--builtin-profile', '-p', f'Font={s.font_family},{fmt_size(s.font_size)}', '-p', f'TerminalColumns={s.columns}', '-p', f'TerminalRows={s.lines}', '-e', ] + cmd def ptyxis_cmd(d: str, cmd: list[str], s: Settings) -> list[str]: # ptyxis uses GSettings, the run environment sets GSETTINGS_BACKEND=keyfile # with XDG_CONFIG_HOME pointing into d, so this is where its settings are read from write_file( os.path.join(d, 'xdg', 'config', 'glib-2.0', 'settings', 'keyfile'), '\n'.join( ( '[org/gnome/Ptyxis]', 'use-system-font=false', f"font-name='{s.font_family} {fmt_size(s.font_size)}'", f'default-columns=uint32 {s.columns}', f'default-rows=uint32 {s.lines}', 'restore-window-size=false', 'restore-session=false', '', ) ), ) return ['ptyxis', '--standalone', '--'] + cmd TMUX_SOCKET = 'kitty-comparative-benchmark' def alacritty_tmux_cmd(d: str, cmd: list[str], s: Settings) -> list[str]: return ['alacritty', '--config-file', alacritty_config(d, s), '-e', 'tmux', '-L', TMUX_SOCKET, '-f', '/dev/null', 'new-session', '--'] + cmd def tmux_cleanup(d: str) -> None: subprocess.run(['tmux', '-L', TMUX_SOCKET, 'kill-server'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) def kitty_exe() -> str: q = os.path.join(BASE, 'kitty', 'launcher', 'kitty') return q if os.access(q, os.X_OK) else 'kitty' def kitten_exe() -> str: q = os.path.join(BASE, 'kitty', 'launcher', 'kitten') if os.access(q, os.X_OK): return q ans = shutil.which('kitten') if not ans: raise SystemExit('Could not find the kitten executable, build kitty first with: make debug') return ans def all_terminals() -> list[Terminal]: return [ Terminal('xterm', 'xterm', xterm_cmd, ['xterm', '-version'], needs_x11=True), Terminal('foot', 'foot', foot_cmd, ['foot', '--version']), Terminal('kitty', kitty_exe(), kitty_cmd, [kitty_exe(), '--version']), Terminal('ghostty', 'ghostty', ghostty_cmd, ['ghostty', '--version']), Terminal('alacritty', 'alacritty', alacritty_cmd, ['alacritty', '--version']), Terminal('wezterm', 'wezterm', wezterm_cmd, ['wezterm', '--version']), Terminal('konsole', 'konsole', konsole_cmd, ['konsole', '--version']), Terminal('ptyxis', 'ptyxis', ptyxis_cmd, ['ptyxis', '--version'], needs_session=True), Terminal('alacritty+tmux', 'alacritty', alacritty_tmux_cmd, ['tmux', '-V'], tmux_cleanup), ] def is_installed(t: Terminal) -> bool: if t.name == 'alacritty+tmux' and not shutil.which('tmux'): return False return bool(shutil.which(t.executable)) def terminal_version(t: Terminal) -> str: if not t.version_cmd: return '' try: cp = subprocess.run(t.version_cmd, capture_output=True, text=True, timeout=10) except (OSError, subprocess.TimeoutExpired) as e: return f'unknown ({e})' lines = [x.strip() for x in (cp.stdout or cp.stderr).splitlines() if x.strip()] ans = lines[0] if lines else 'unknown' if t.name == 'alacritty+tmux': a = Terminal('alacritty', 'alacritty', alacritty_cmd, ['alacritty', '--version']) ans = f'{terminal_version(a)} + {ans}' return ans # }}} # CPU setup {{{ def read_sysfs(path: str) -> str: try: with open(path) as f: return f.read().strip() except OSError: return '' def online_cpus() -> list[int]: return sorted(os.sched_getaffinity(0)) # Per CPU sysfs files that indicate relative CPU performance: cpu_capacity (ARM # big.LITTLE, constant on x86), CPPC highest perf (AMD and Intel) and max frequency PERF_METRICS = ('cpu_capacity', 'cpufreq/amd_pstate_highest_perf', 'acpi_cppc/highest_perf', 'cpufreq/cpuinfo_max_freq') def physical_core_key(cpu: int) -> tuple[str, str]: base = f'{CPU_SYSFS}/cpu{cpu}/topology' return read_sysfs(f'{base}/physical_package_id'), read_sysfs(f'{base}/core_id') def detect_fast_cpus(one_thread_per_core: bool) -> tuple[list[int], str]: cpus = online_cpus() intel_core = read_sysfs('/sys/devices/cpu_core/cpus') fast: list[int] if intel_core: # Intel hybrid, P-cores fast = [c for c in parse_cpu_list(intel_core) if c in cpus] how = 'Intel hybrid P-cores' else: for q in PERF_METRICS: vals = {c: read_sysfs(f'{CPU_SYSFS}/cpu{c}/{q}') for c in cpus} if not all(v.isdigit() for v in vals.values()): continue metrics = {c: int(v) for c, v in vals.items()} top = max(metrics.values()) # Ignore small per-core variations from binning, e.g. AMD preferred cores fast = [c for c in cpus if metrics[c] >= 0.9 * top] if len(fast) < len(cpus): how = f'heterogeneous cores, using the fastest as per {q}' break else: return cpus, 'homogeneous cores, using all CPUs' if one_thread_per_core: seen: set[tuple[str, str]] = set() ans = [] for c in fast: k = physical_core_key(c) if k not in seen: seen.add(k) ans.append(c) fast, how = ans, how + ', one thread per core' return fast, how def parse_cpu_list(spec: str) -> list[int]: ans: list[int] = [] for part in spec.split(','): part = part.strip() if not part: continue if '-' in part: a, b = part.split('-', 1) ans.extend(range(int(a), int(b) + 1)) else: ans.append(int(part)) return ans def write_sysfs_values(values: dict[str, str]) -> str: """Write the values, using sudo if needed. Returns an error message on failure.""" if not values: return '' try: for path, val in values.items(): with open(path, 'w') as f: f.write(val) return '' except PermissionError: pass except OSError as e: return str(e) script = '\n'.join(f'echo {shlex.quote(v)} > {shlex.quote(p)}' for p, v in values.items()) cp = subprocess.run(['sudo', '-n', 'sh', '-c', script], capture_output=True, text=True) if cp.returncode == 0: return '' if sys.stdin.isatty(): print('Root access is needed to change the CPU frequency governor, asking sudo for your password', file=sys.stderr) cp = subprocess.run(['sudo', 'sh', '-c', script], stdout=subprocess.PIPE, text=True) if cp.returncode == 0: return '' return 'permission denied: not root and passwordless sudo is not available' @contextmanager def performance_governor(enabled: bool) -> Generator[str]: """Yields a description of what was done""" if not enabled: yield 'not changed (disabled on command line)' return original: dict[str, str] = {} for cpu in online_cpus(): p = f'{CPU_SYSFS}/cpu{cpu}/cpufreq/scaling_governor' if cur := read_sysfs(p): avail = read_sysfs(f'{CPU_SYSFS}/cpu{cpu}/cpufreq/scaling_available_governors').split() if 'performance' not in avail: yield f'not changed: the performance governor is not available (available: {" ".join(avail)})' return original[p] = cur if not original: yield 'not changed: no cpufreq support' return to_change = {p: 'performance' for p, v in original.items() if v != 'performance'} if not to_change: yield 'performance (already set)' return if err := write_sysfs_values(to_change): yield f'not changed ({", ".join(sorted(set(original.values())))}): {err}' return try: yield f'performance (was {", ".join(sorted(set(original.values())))}, will be restored)' finally: if err := write_sysfs_values({p: original[p] for p in to_change}): print(f'Failed to restore the CPU frequency governor: {err}', file=sys.stderr) # }}} # Sway support for making window sizes identical {{{ def sway_available() -> bool: return bool(os.environ.get('SWAYSOCK')) and bool(shutil.which('swaymsg')) def sway_find_window(pid: int, marker: str) -> int | None: try: cp = subprocess.run(['swaymsg', '-t', 'get_tree', '-r'], capture_output=True, text=True, check=True, timeout=10) except (OSError, subprocess.SubprocessError): return None tree: dict[str, Any] = json.loads(cp.stdout) stack = [tree] while stack: n = stack.pop() if n.get('type') in ('con', 'floating_con') and (n.get('pid') == pid or marker in (n.get('name') or '')): con_id = n.get('id') if isinstance(con_id, int): return con_id stack.extend(n.get('nodes', ())) stack.extend(n.get('floating_nodes', ())) return None def sway_float(pid: int, marker: str, width: int, height: int, timeout: float = 15) -> str: """Make the window floating, so that it gets its natural (requested) size, unless width and height are non-zero in which case it is resized to them.""" deadline = time.monotonic() + timeout while (con_id := sway_find_window(pid, marker)) is None: if time.monotonic() > deadline: return 'could not find terminal window in the sway tree' time.sleep(0.1) resize = ['resize', 'set', 'width', f'{width}px', 'height', f'{height}px,'] if width and height else [] cp = subprocess.run( ['swaymsg', f'[con_id={con_id}]', 'floating', 'enable,'] + resize + ['move', 'position', '0', '0,', 'focus'], capture_output=True, text=True ) if cp.returncode != 0: return f'swaymsg failed: {cp.stdout.strip()} {cp.stderr.strip()}' return '' # }}} # Helpers for running terminals in minimal sessions {{{ @contextmanager def private_xwayland(width: int, height: int) -> Generator[tuple[str, int]]: """Start a rootful Xwayland, yields (DISPLAY, pid)""" if not os.environ.get('WAYLAND_DISPLAY') or not shutil.which('Xwayland'): raise RuntimeError('DISPLAY is not set and Xwayland is not available to create one') r, w = os.pipe() try: proc = subprocess.Popen( ['Xwayland', '-displayfd', str(w), '-geometry', f'{width}x{height}', '-nolisten', 'tcp'], pass_fds=(w,), stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, start_new_session=True, ) finally: os.close(w) try: with open(r, 'rb') as f: num = f.readline().strip().decode() if not num.isdigit(): raise RuntimeError(f'Xwayland failed to start, exit code: {proc.poll()}') yield f':{num}', proc.pid finally: kill_process_group(proc) def has_session_bus() -> bool: rd = os.environ.get('XDG_RUNTIME_DIR', '') return bool(os.environ.get('DBUS_SESSION_BUS_ADDRESS')) or (bool(rd) and os.path.exists(os.path.join(rd, 'bus'))) def has_systemd_user_instance() -> bool: if not shutil.which('systemctl'): return False return subprocess.run(['systemctl', '--user', 'show-environment'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode == 0 SYSTEMD_RUN_SHIM = """#!/bin/sh # Used when there is no systemd user instance: ignore all options and run the command directly while [ $# -gt 0 ]; do case "$1" in --) shift; break;; -*) shift;; *) break;; esac; done exec "$@" """ def setup_session(d: str, cmd: list[str], env: dict[str, str]) -> list[str]: """Make up for a missing session bus and/or systemd user instance, as found in headless sessions""" if not has_systemd_user_instance(): shim = write_file(os.path.join(d, 'shim', 'systemd-run'), SYSTEMD_RUN_SHIM) os.chmod(shim, 0o755) env['PATH'] = os.path.dirname(shim) + os.pathsep + env.get('PATH', '') if not has_session_bus(): if not shutil.which('dbus-run-session'): raise RuntimeError('there is no D-Bus session bus and dbus-run-session is not available to create one') env['GTK_A11Y'] = 'none' # the accessibility bus is not available in a private session cmd = ['dbus-run-session', '--'] + cmd return cmd # }}} # Running a single terminal {{{ @dataclass class BenchmarkResult: name: str status: str = 'ok' # ok, timeout or failed error: str = '' description: str = '' duration: float = 0 data_size: int = 0 repetitions: int = 0 mb_per_second: float = 0 cells_per_second: float = 0 @dataclass class TerminalRun: terminal: str version: str results: list[BenchmarkResult] = field(default_factory=list) error: str = '' size: str = '' size_warning: str = '' wall_time: float = 0 def clean_env(xdg_root: str) -> dict[str, str]: env = {k: v for k, v in os.environ.items() if not k.startswith(ENV_PREFIXES_TO_REMOVE)} env.pop('TERM', None) for name in ('CONFIG', 'DATA', 'STATE', 'CACHE'): p = os.path.join(xdg_root, name.lower()) os.makedirs(p, exist_ok=True) env[f'XDG_{name}_HOME'] = p env['GSETTINGS_BACKEND'] = 'keyfile' return env def wait_for(predicate: Callable[[], bool], proc: subprocess.Popen[bytes], timeout: float, on_poll: Callable[[], None] | None = None) -> str: deadline = time.monotonic() + timeout while not predicate(): if time.monotonic() > deadline: return f'timed out after {timeout:.0f} seconds' if (rc := proc.poll()) is not None and not predicate(): # some launchers fork and exit, so give the real process a chance time.sleep(0.5) if predicate(): break if rc != 0: return f'terminal process exited with code {rc}' if on_poll is not None: on_poll() time.sleep(0.05) return '' def kill_process_group(proc: subprocess.Popen[bytes]) -> None: for sig in (signal.SIGTERM, signal.SIGKILL): with suppress(ProcessLookupError, PermissionError): os.killpg(proc.pid, sig) try: proc.wait(timeout=3) return except subprocess.TimeoutExpired: continue def available_benchmarks() -> list[str]: cp = subprocess.run([kitten_exe(), '__benchmark__', '--help'], stdin=subprocess.DEVNULL, capture_output=True, text=True) ans = re.findall(r'^\s*\*\s+(\w+):', cp.stdout, flags=re.MULTILINE) if not ans: raise SystemExit(f'Failed to get the list of benchmarks from the benchmark kitten: {cp.stderr}') return ans def load_result(name: str, info: dict[str, Any], path: str) -> BenchmarkResult: status = str(info.get('status', 'failed')) if status != 'ok': return BenchmarkResult(name, status=status, error=str(info.get('error', ''))) with open(path) as f: raw: list[dict[str, Any]] = json.load(f) r = raw[0] return BenchmarkResult( name=str(r['name']), description=str(r['description']), duration=float(r['duration_seconds']), data_size=int(r['data_size_bytes']), repetitions=int(r['repetitions']), mb_per_second=float(r['mb_per_second']), cells_per_second=float(r.get('cells_per_second', 0)), ) def check_size(size: str, settings: Settings) -> str: if settings.window_width or not size: return '' # forced pixel size, so the number of cells is expected to differ cols, lines = map(int, size.split('x')) # allow for off by one because of padding, and the status line in tmux if abs(cols - settings.columns) > 1 or abs(lines - settings.lines) > 1: return f'window size is {size} cells instead of {settings.columns}x{settings.lines}' return '' def run_terminal( t: Terminal, args: argparse.Namespace, settings: Settings, cpus: list[int], log: Callable[[str], None], progress: Callable[[str], None] ) -> TerminalRun: ans = TerminalRun(t.name, terminal_version(t)) with tempfile.TemporaryDirectory(prefix='kitty-cbench-') as d: marker = f'cbench-{os.getpid()}-{t.name}' inner = [sys.executable, os.path.abspath(__file__), '--inner', d, marker, kitten_exe(), str(args.repetitions), str(args.benchmark_timeout)] + list( args.benchmarks ) cmd = t.build_cmd(d, inner, settings) env = clean_env(os.path.join(d, 'xdg')) stack = ExitStack() window_pid = 0 try: if t.needs_x11 and not env.get('DISPLAY'): env['DISPLAY'], window_pid = stack.enter_context(private_xwayland(settings.window_width or 1600, settings.window_height or 1000)) if t.needs_session: cmd = setup_session(d, cmd, env) except RuntimeError as e: stack.close() ans.error = str(e) return ans log(f'Running: {shlex.join(cmd)}') start = time.monotonic() try: proc = subprocess.Popen( cmd, env=env, cwd=d, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, start_new_session=True, preexec_fn=lambda: os.sched_setaffinity(0, cpus), ) except OSError as e: stack.close() ans.error = f'failed to launch: {e}' return ans try: ans.error = run_in_launched_terminal(proc, window_pid or proc.pid, d, marker, args, settings, progress) if not ans.error: with open(os.path.join(d, DONE_FILE)) as f: done: dict[str, Any] = json.load(f) ans.size = str(done.get('size', '')) ans.size_warning = check_size(ans.size, settings) statuses: dict[str, dict[str, Any]] = done['benchmarks'] ans.results = [load_result(name, info, os.path.join(d, f'results-{name}.json')) for name, info in statuses.items()] except (OSError, ValueError, KeyError, IndexError) as e: ans.error = f'failed to read results: {e}' finally: ans.wall_time = time.monotonic() - start kill_process_group(proc) if t.cleanup is not None: t.cleanup(d) stack.close() return ans def run_in_launched_terminal( proc: subprocess.Popen[bytes], window_pid: int, d: str, marker: str, args: argparse.Namespace, settings: Settings, progress: Callable[[str], None], ) -> str: if err := wait_for(lambda: os.path.exists(os.path.join(d, READY_FILE)), proc, 60): return f'terminal did not start: {err}' if args.sway: if err := sway_float(window_pid, marker, settings.window_width, settings.window_height): return f'could not resize the window: {err}' time.sleep(0.5) # let the terminal process the resize with open(os.path.join(d, GO_FILE), 'w'): pass progress_file = os.path.join(d, PROGRESS_FILE) last_progress = '' def report_progress() -> None: nonlocal last_progress try: with open(progress_file) as f: current = f.read() except FileNotFoundError: return if current and current != last_progress: last_progress = current progress(current) # The inner process enforces the per benchmark timeout, allow some extra time for startup and cleanup timeout = (args.benchmark_timeout + 30) * len(args.benchmarks) + 60 if err := wait_for(lambda: os.path.exists(os.path.join(d, DONE_FILE)), proc, timeout, report_progress): return f'benchmark did not complete: {err}' return '' def write_atomic(path: str, data: str) -> None: tmp = path + '.tmp' with open(tmp, 'w') as f: f.write(data) os.replace(tmp, path) def recover_tty_after_kill(tty_fd: int) -> None: import termios # Wait for the terminal to process whatever is left in the pty buffer, reset # it and then discard any responses it sent to queries os.write(tty_fd, b'\x1b[?2026l\x1b[m\x1bc') time.sleep(1) termios.tcflush(tty_fd, termios.TCIFLUSH) def inner_main(argv: list[str]) -> None: """Runs inside the terminal being benchmarked""" d, marker, kitten, repetitions, timeout_str, *benchmarks = argv timeout = float(timeout_str) tty_fd = os.open('/dev/tty', os.O_RDWR | os.O_CLOEXEC) os.write(tty_fd, f'\x1b]2;{marker}\x1b\\Waiting to start benchmark...\r\n'.encode()) write_atomic(os.path.join(d, READY_FILE), json.dumps({'pid': os.getpid()})) go = os.path.join(d, GO_FILE) while not os.path.exists(go): time.sleep(0.05) size = os.get_terminal_size(tty_fd) statuses: dict[str, dict[str, str]] = {} for name in benchmarks: write_atomic(os.path.join(d, PROGRESS_FILE), name) with open(os.path.join(d, f'log-{name}.txt'), 'w+') as log: p = subprocess.Popen( [kitten, '__benchmark__', '--repetitions', repetitions, '--output', os.path.join(d, f'results-{name}.json'), name], stdout=log, stderr=subprocess.STDOUT, ) try: rc = p.wait(timeout=timeout) except subprocess.TimeoutExpired: p.kill() p.wait() recover_tty_after_kill(tty_fd) statuses[name] = {'status': 'timeout', 'error': f'took longer than {timeout:.0f} seconds'} continue if rc == 0: statuses[name] = {'status': 'ok'} else: log.seek(0) statuses[name] = {'status': 'failed', 'error': f'exit code {rc}: {log.read()[-1000:].strip()}'} write_atomic(os.path.join(d, DONE_FILE), json.dumps({'benchmarks': statuses, 'size': f'{size.columns}x{size.lines}'})) # }}} # Output {{{ class Style: def __init__(self, enabled: bool) -> None: self.enabled = enabled def __call__(self, text: str, *codes: int) -> str: if not self.enabled or not codes: return text return f'\x1b[{";".join(map(str, codes))}m{text}\x1b[m' def bold(self, text: str) -> str: return self(text, 1) def dim(self, text: str) -> str: return self(text, 2) def green(self, text: str) -> str: return self(text, 32) def red(self, text: str) -> str: return self(text, 31) def yellow(self, text: str) -> str: return self(text, 33) def cyan(self, text: str) -> str: return self(text, 36) ANSI_PAT = re.compile(r'\x1b\[[0-9;]*m') def visible_len(text: str) -> int: return len(ANSI_PAT.sub('', text)) def pad(text: str, width: int, right: bool = True) -> str: p = ' ' * max(0, width - visible_len(text)) return p + text if right else text + p def table(header: Sequence[str], rows: Sequence[Sequence[str]], style: Style) -> str: widths = [max(visible_len(r[i]) for r in [header, *rows]) for i in range(len(header))] lines = [' '.join(pad(style.bold(h), w, i > 0) for i, (h, w) in enumerate(zip(header, widths)))] lines.append(style.dim(' '.join('─' * w for w in widths))) for r in rows: lines.append(' '.join(pad(c, w, i > 0) for i, (c, w) in enumerate(zip(r, widths)))) return '\n'.join(lines) def fmt_cells(x: float) -> str: return f'{x / 1e6:.1f}M' if x else '-' def fmt_duration(x: float) -> str: return f'{x:.2f}s' if x < 60 else f'{int(x // 60)}m{x % 60:04.1f}s' def print_terminal_run(r: TerminalRun, style: Style) -> None: title = style.bold(style.cyan(r.terminal)) meta = ', '.join(x for x in (r.version, f'{r.size} cells' if r.size else '', f'took {fmt_duration(r.wall_time)}') if x) print(f'{title} {style.dim(f"({meta})")}') if r.size_warning: print(style.yellow(f' WARNING: {r.size_warning}')) if r.error: print(style.red(f' FAILED: {r.error}')) print() return rows = [] for b in r.results: if b.status == 'ok': rows.append([b.name, f'{b.mb_per_second:.1f}', fmt_cells(b.cells_per_second), fmt_duration(b.duration), f'{b.data_size / (1024 * 1024):.0f} MB']) else: rows.append([b.name, style.red(b.status), '', '', '']) print('\n'.join(' ' + x for x in table(['Benchmark', 'MB/s', 'Cells/s', 'Time', 'Data'], rows, style).splitlines())) for b in r.results: if b.status != 'ok': print(style.red(f' {b.name} {b.status}: {b.error}')) print() def print_summary(runs: list[TerminalRun], style: Style) -> None: ok = [r for r in runs if r.results and not r.error] if not ok: return names: list[str] = [] for r in ok: for b in r.results: if b.name not in names: names.append(b.name) print( style.bold('Summary: throughput in MB/s, higher is better. ') + style.dim('Fastest in green, slowest in red, (n×) is how many times slower than the fastest.') ) rows = [] for name in names: vals = {r.terminal: b.mb_per_second for r in ok for b in r.results if b.name == name and b.status == 'ok'} statuses = {r.terminal: b.status for r in ok for b in r.results if b.name == name} best, worst = max(vals.values(), default=0), min(vals.values(), default=0) row = [name] for r in ok: v = vals.get(r.terminal) if v is None: row.append(style.red(statuses.get(r.terminal, '-'))) continue text = f'{v:.1f}' if v == best: text = style.bold(style.green(text)) elif len(vals) > 2 and v == worst: text = style.red(text) if v != best and v > 0: text += style.dim(f' ({best / v:.1f}×)') row.append(text) rows.append(row) print(table(['Benchmark'] + [r.terminal for r in ok], rows, style)) print() # }}} def parse_args() -> argparse.Namespace: terminal_names = [t.name for t in all_terminals()] p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) p.add_argument('benchmarks', nargs='*', help='Benchmarks to run, by default all are run. See kitten __benchmark__ --help for the list') p.add_argument( '--terminals', '-t', default=','.join(terminal_names), help=f'Comma separated list of terminals to benchmark, from: {", ".join(terminal_names)}' ) p.add_argument('--repetitions', '-r', type=int, default=500, help='The number of repetitions of each benchmark. Default: %(default)s') p.add_argument('--font-family', default='DejaVu Sans Mono', help='Font family to use in all terminals. Default: %(default)s') p.add_argument('--font-size', type=float, default=11, help='Font size in pts to use in all terminals. Default: %(default)s') p.add_argument('--columns', type=int, default=120, help='Initial window width in cells. Default: %(default)s') p.add_argument('--lines', type=int, default=40, help='Initial window height in cells. Default: %(default)s') p.add_argument( '--window-size', default='', help='Window size in pixels, used only when running under sway. Under sway, windows are made floating so that tiling does not' ' override the initial size in cells. If this is specified they are additionally resized to it.', ) p.add_argument('--cpus', default='', help='Comma separated list of CPUs to pin the terminals to, overrides automatic fast core detection') p.add_argument('--all-smt-threads', action='store_true', help='Use all SMT threads of the fast cores instead of one per core') p.add_argument('--no-governor', action='store_true', help='Do not change the CPU frequency governor') p.add_argument('--no-sway', action='store_true', help='Do not use sway to make window sizes identical, even if running under sway') p.add_argument( '--benchmark-timeout', type=float, default=180, help='Max time in seconds for each benchmark in each terminal, after which it is aborted and reported as timed out. Default: %(default)s', ) p.add_argument('--output', '-o', default='', help='Also write all results as JSON to the specified file') p.add_argument('--verbose', '-v', action='store_true', help='Print the command lines used to run the terminals') args = p.parse_args() args.sway = sway_available() and not args.no_sway if args.benchmarks: avail = available_benchmarks() if unknown := [x for x in args.benchmarks if x not in avail]: raise SystemExit(f'Unknown benchmarks: {", ".join(unknown)}. Choose from: {", ".join(avail)}') else: args.benchmarks = available_benchmarks() return args def main() -> None: if len(sys.argv) > 1 and sys.argv[1] == '--inner': inner_main(sys.argv[2:]) return if not sys.platform.startswith('linux'): raise SystemExit('This script only works on Linux') args = parse_args() # Ensure cleanup code (killing terminals, restoring the governor) runs when terminated signal.signal(signal.SIGTERM, lambda signum, frame: sys.exit(128 + signum)) style = Style(sys.stdout.isatty()) err_style = Style(sys.stderr.isatty()) wanted = [x.strip() for x in args.terminals.split(',') if x.strip()] known = {t.name: t for t in all_terminals()} if unknown := [x for x in wanted if x not in known]: raise SystemExit(f'Unknown terminals: {", ".join(unknown)}') terminals = [known[x] for x in wanted] missing = [t.name for t in terminals if not is_installed(t)] terminals = [t for t in terminals if is_installed(t)] if not terminals: raise SystemExit('None of the terminals are installed') ww = wh = 0 if args.window_size: try: ww, wh = map(int, args.window_size.lower().split('x')) except ValueError: raise SystemExit(f'Invalid window size: {args.window_size}') settings = Settings(args.font_family, args.font_size, args.columns, args.lines, ww, wh) kitten_exe() # fail early if not found if args.cpus: cpus, cpu_how = parse_cpu_list(args.cpus), 'specified on command line' else: cpus, cpu_how = detect_fast_cpus(not args.all_smt_threads) if shutil.which('fc-list'): fc = subprocess.run(['fc-list', f'{settings.font_family}:spacing=mono', 'family'], capture_output=True, text=True) if not fc.stdout.strip(): print( err_style.yellow(f'Warning: the font {settings.font_family!r} does not seem to be installed, terminals will use fallback fonts'), file=sys.stderr, ) def log(msg: str) -> None: if args.verbose: print(err_style.dim(msg), file=sys.stderr) runs: list[TerminalRun] = [] with performance_governor(not args.no_governor) as governor: print(style.bold('Comparative terminal benchmark')) print(f' CPUs: {",".join(map(str, cpus))} ({cpu_how})') print(f' Governor: {governor}') print(f' Font: {settings.font_family} {fmt_size(settings.font_size)}pt') win = f'{settings.columns}x{settings.lines} cells' if args.sway: win += f', resized to {ww}x{wh} pixels' if ww else ', floating sway window' print(f' Window: {win}') print(f' Reps: {args.repetitions} (timeout of {args.benchmark_timeout:.0f}s per benchmark)') if missing: print(f' Skipped: {", ".join(missing)} (not installed)') print() sys.stdout.flush() for t in terminals: print(err_style.dim(f'Benchmarking {t.name}...'), file=sys.stderr) r = run_terminal(t, args, settings, cpus, log, lambda name: print(err_style.dim(f' {name}'), file=sys.stderr)) runs.append(r) print_terminal_run(r, style) sys.stdout.flush() print_summary(runs, style) if args.output: with open(args.output, 'w') as f: json.dump( [ { 'terminal': r.terminal, 'version': r.version, 'size': r.size, 'size_warning': r.size_warning, 'error': r.error, 'wall_time': r.wall_time, 'results': [b.__dict__ for b in r.results], } for r in runs ], f, indent=2, ) if __name__ == '__main__': main()