kitty/tools/utils/mime_cache.go
Kovid Goyal a5c75acb7d
Some checks are pending
CI / Linux (python=3.14 cc=clang sanitize=1) (push) Waiting to run
CI / Linux (python=3.12 cc=gcc sanitize=0) (push) Waiting to run
CI / Linux (python=3.13 cc=gcc sanitize=1) (push) Waiting to run
CI / Linux package (push) Waiting to run
CI / Bundle test (macos-latest) (push) Waiting to run
CI / Bundle test (ubuntu-latest) (push) Waiting to run
CI / macOS Brew (push) Waiting to run
CI / Test ./dev.sh and benchmark (push) Waiting to run
CodeQL / CodeQL-Build (actions, ubuntu-latest) (push) Waiting to run
CodeQL / CodeQL-Build (c, macos-latest) (push) Waiting to run
CodeQL / CodeQL-Build (c, ubuntu-latest) (push) Waiting to run
CodeQL / CodeQL-Build (go, ubuntu-latest) (push) Waiting to run
CodeQL / CodeQL-Build (python, ubuntu-latest) (push) Waiting to run
Depscan / Scan dependencies for vulnerabilities (push) Waiting to run
handle invalid UTF-8 in binary mime db
2026-10-06 08:58:50 +05:30

493 lines
13 KiB
Go

// License: GPLv3 Copyright: 2026, Kovid Goyal, <kovid at kovidgoyal.net>
package utils
import (
"bytes"
"encoding/binary"
"errors"
"fmt"
"io/fs"
"os"
"path"
"path/filepath"
"runtime"
"strings"
"sync"
"golang.org/x/sys/unix"
)
// Fast lookup of MIME types for file names using the binary mime.cache files
// generated by update-mime-database from shared-mime-info. The lookup
// algorithm mirrors the one used by xdgmime (GLib/GIO). See
// https://specifications.freedesktop.org/shared-mime-info-spec/latest/ar01s02.html
const (
mime_cache_case_sensitive_flag = 0x100
mime_cache_weight_mask = 0xff
mime_cache_max_matches = 16
mime_cache_max_suffix_tree_depth = 256
mime_cache_header_size = 40
mime_cache_alias_list_offset_pos = 4
mime_cache_literal_list_offset_pos = 12
mime_cache_suffix_tree_offset_pos = 16
mime_cache_glob_list_offset_pos = 20
mime_cache_default_data_dirs = "/usr/local/share/:/usr/share/"
mime_cache_relative_path_in_datadir = "mime/mime.cache"
)
type mime_cache struct {
data []byte
}
type mime_cache_match struct {
mime string
weight uint32
}
type mime_cache_matches struct {
items []mime_cache_match
}
func (m *mime_cache_matches) full() bool { return len(m.items) >= mime_cache_max_matches }
func (c *mime_cache) u32(offset uint64) (uint32, bool) {
if offset > uint64(len(c.data)) || uint64(len(c.data))-offset < 4 {
return 0, false
}
return binary.BigEndian.Uint32(c.data[offset:]), true
}
func (c *mime_cache) entry(offset uint64) (a, b, weight uint32, ok bool) {
if a, ok = c.u32(offset); !ok {
return
}
if b, ok = c.u32(offset + 4); !ok {
return
}
weight, ok = c.u32(offset + 8)
return
}
// cstring returns the NUL terminated string at offset, without copying
func (c *mime_cache) cstring(offset uint32) ([]byte, bool) {
if uint64(offset) >= uint64(len(c.data)) {
return nil, false
}
data := c.data[offset:]
idx := bytes.IndexByte(data, 0)
if idx < 0 {
return nil, false
}
return data[:idx], true
}
func (c *mime_cache) add_match(ans *mime_cache_matches, mime_offset, weight uint32) bool {
if ans.full() {
return false
}
if mime, ok := c.cstring(mime_offset); ok {
ans.items = append(ans.items, mime_cache_match{string(mime), weight & mime_cache_weight_mask})
return true
}
return false
}
func (c *mime_cache) lookup_literal(name []byte, case_sensitive_check bool, ans *mime_cache_matches) bool {
list_offset, ok := c.u32(mime_cache_literal_list_offset_pos)
if !ok {
return false
}
n, ok := c.u32(uint64(list_offset))
if !ok {
return false
}
lo, hi := uint64(0), uint64(n)
for lo < hi {
mid := lo + (hi-lo)/2
literal_offset, mime_offset, weight, ok := c.entry(uint64(list_offset) + 4 + 12*mid)
if !ok {
return false
}
literal, ok := c.cstring(literal_offset)
if !ok {
return false
}
switch cmp := bytes.Compare(literal, name); {
case cmp < 0:
lo = mid + 1
case cmp > 0:
hi = mid
default:
if !case_sensitive_check && weight&mime_cache_case_sensitive_flag != 0 {
return false
}
return c.add_match(ans, mime_offset, weight)
}
}
return false
}
func (c *mime_cache) lookup_suffix_node(num_nodes, offset uint32, name []rune, case_sensitive_check bool, ans *mime_cache_matches, depth int) {
if len(name) == 0 || depth > mime_cache_max_suffix_tree_depth {
return
}
ch := uint32(name[len(name)-1])
lo, hi := uint64(0), uint64(num_nodes)
for lo < hi {
mid := lo + (hi-lo)/2
node_char, num_children, first_child_offset, ok := c.entry(uint64(offset) + 12*mid)
if !ok {
return
}
switch {
case ch < node_char:
hi = mid
case ch > node_char:
lo = mid + 1
default:
before := len(ans.items)
c.lookup_suffix_node(num_children, first_child_offset, name[:len(name)-1], case_sensitive_check, ans, depth+1)
if len(ans.items) == before {
// No longer suffix matched, use the leaf nodes of this node,
// which are sorted before all other children.
for i := range uint64(num_children) {
leaf_char, mime_offset, weight, ok := c.entry(uint64(first_child_offset) + 12*i)
if !ok || leaf_char != 0 {
break
}
if case_sensitive_check || weight&mime_cache_case_sensitive_flag == 0 {
c.add_match(ans, mime_offset, weight)
}
}
}
return
}
}
}
func (c *mime_cache) lookup_suffix(name []rune, case_sensitive_check bool, ans *mime_cache_matches) {
tree_offset, ok := c.u32(mime_cache_suffix_tree_offset_pos)
if !ok {
return
}
num_roots, ok := c.u32(uint64(tree_offset))
if !ok {
return
}
first_root_offset, ok := c.u32(uint64(tree_offset) + 4)
if !ok {
return
}
c.lookup_suffix_node(num_roots, first_root_offset, name, case_sensitive_check, ans, 0)
}
// fnmatch_to_go_pattern converts the fnmatch() negated bracket expression
// syntax [!...] to the [^...] syntax used by path.Match
func fnmatch_to_go_pattern(glob string) string {
if !strings.Contains(glob, "[!") {
return glob
}
return strings.ReplaceAll(glob, "[!", "[^")
}
func (c *mime_cache) lookup_glob(name string, case_sensitive_check bool, ans *mime_cache_matches) {
list_offset, ok := c.u32(mime_cache_glob_list_offset_pos)
if !ok {
return
}
n, ok := c.u32(uint64(list_offset))
if !ok {
return
}
for i := uint64(0); i < uint64(n) && !ans.full(); i++ {
glob_offset, mime_offset, weight, ok := c.entry(uint64(list_offset) + 4 + 12*i)
if !ok {
return
}
if !case_sensitive_check && weight&mime_cache_case_sensitive_flag != 0 {
continue
}
if glob, ok := c.cstring(glob_offset); ok {
if matched, err := path.Match(fnmatch_to_go_pattern(string(glob)), name); err == nil && matched {
c.add_match(ans, mime_offset, weight)
}
}
}
}
type mime_caches []*mime_cache
func (caches mime_caches) lookup(name string) string {
if name == "" || strings.IndexByte(name, 0) > -1 {
return ""
}
ans := mime_cache_matches{items: make([]mime_cache_match, 0, 4)}
bname := []byte(name)
for _, c := range caches {
if c.lookup_literal(bname, true, &ans) {
return ans.items[0].mime
}
}
lname := strings.ToLower(name)
for _, c := range caches {
if c.lookup_literal([]byte(lname), false, &ans) {
return ans.items[0].mime
}
}
lrunes := []rune(lname)
for _, c := range caches {
c.lookup_suffix(lrunes, false, &ans)
}
if len(ans.items) < 2 {
runes := []rune(name)
for _, c := range caches {
c.lookup_suffix(runes, true, &ans)
}
}
if len(ans.items) == 0 {
for _, c := range caches {
c.lookup_glob(name, true, &ans)
}
}
if len(ans.items) == 0 {
for _, c := range caches {
c.lookup_glob(lname, false, &ans)
}
}
if len(ans.items) == 0 {
return ""
}
// Highest weight wins, ties go to the earliest match, which means the
// longest suffix and the highest priority cache file.
best := ans.items[0]
for _, m := range ans.items[1:] {
if m.weight > best.weight {
best = m
}
}
return best.mime
}
func (c *mime_cache) alias_list() (list_offset, n uint32, ok bool) {
if list_offset, ok = c.u32(mime_cache_alias_list_offset_pos); ok {
n, ok = c.u32(uint64(list_offset))
}
return
}
func (c *mime_cache) alias_entry(list_offset uint32, idx uint64) (alias, mime []byte, ok bool) {
offset := uint64(list_offset) + 4 + 8*idx
alias_offset, ok := c.u32(offset)
if !ok {
return
}
mime_offset, ok := c.u32(offset + 4)
if !ok {
return
}
if alias, ok = c.cstring(alias_offset); ok {
mime, ok = c.cstring(mime_offset)
}
return
}
// canonical_mime_type returns the canonical MIME type for the specified alias
// or the empty string if mime is not an alias. The alias list is sorted by alias.
func (c *mime_cache) canonical_mime_type(mime []byte) string {
list_offset, n, ok := c.alias_list()
if !ok {
return ""
}
lo, hi := uint64(0), uint64(n)
for lo < hi {
mid := lo + (hi-lo)/2
alias, canonical, ok := c.alias_entry(list_offset, mid)
if !ok {
return ""
}
switch cmp := bytes.Compare(alias, mime); {
case cmp < 0:
lo = mid + 1
case cmp > 0:
hi = mid
default:
return string(canonical)
}
}
return ""
}
func (c *mime_cache) aliases_of(canonical []byte, add func(string)) {
list_offset, n, ok := c.alias_list()
if !ok {
return
}
for i := range uint64(n) {
alias, mime, ok := c.alias_entry(list_offset, i)
if !ok {
return
}
if bytes.Equal(mime, canonical) {
add(string(alias))
}
}
}
func (caches mime_caches) aliases(mime string) (ans []string) {
canonical := ""
for _, c := range caches {
if canonical = c.canonical_mime_type([]byte(mime)); canonical != "" {
break
}
}
if canonical == "" {
canonical = mime
}
seen := NewSet[string](8)
add := func(x string) {
if !seen.Has(x) {
seen.Add(x)
ans = append(ans, x)
}
}
add(canonical)
for _, c := range caches {
c.aliases_of([]byte(canonical), add)
}
add(mime)
return
}
func load_mime_cache(path string) (*mime_cache, error) {
f, err := os.Open(path)
if err != nil {
return nil, err
}
defer f.Close()
st, err := f.Stat()
if err != nil {
return nil, err
}
if !st.Mode().IsRegular() {
return nil, fmt.Errorf("%s is not a regular file", path)
}
if st.Size() < mime_cache_header_size {
return nil, fmt.Errorf("the MIME cache file %s is too small", path)
}
// The mapping is never unmapped as it is used for the lifetime of the
// process. update-mime-database replaces the cache file atomically so the
// mapped data remains valid.
// Note for security reviews: if the file is truncated in place, accessing
// the mapped pages beyond the new end of file raises SIGBUS. This is
// deliberately not handled, as truncating the file in place requires write
// access to the XDG data directories and GLib (xdgmime) and Qt
// (QMimeBinaryProvider) mmap these files in exactly the same way, without
// handling SIGBUS. All reads from the mapped data are bounds checked, so
// corrupt or malicious cache file contents are safe.
data, err := unix.Mmap(int(f.Fd()), 0, int(st.Size()), unix.PROT_READ, unix.MAP_PRIVATE)
is_mmapped := err == nil
if !is_mmapped {
if data, err = os.ReadFile(path); err != nil {
return nil, err
}
if len(data) < mime_cache_header_size {
return nil, fmt.Errorf("the MIME cache file %s is too small", path)
}
}
major, minor := binary.BigEndian.Uint16(data), binary.BigEndian.Uint16(data[2:])
if major != 1 || minor < 1 {
if is_mmapped {
_ = unix.Munmap(data)
}
return nil, fmt.Errorf("the MIME cache file %s has unsupported version %d.%d", path, major, minor)
}
return &mime_cache{data: data}, nil
}
// mime_cache_paths returns the paths to mime.cache files in order of decreasing priority
func mime_cache_paths() (ans []string) {
seen := NewSet[string](4)
add := func(data_dir string) {
// relative paths are ignored as per the XDG spec
if filepath.IsAbs(data_dir) {
if p := filepath.Join(data_dir, mime_cache_relative_path_in_datadir); !seen.Has(p) {
seen.Add(p)
ans = append(ans, p)
}
}
}
if data_home := os.Getenv("XDG_DATA_HOME"); data_home != "" {
add(data_home)
} else if home := os.Getenv("HOME"); home != "" {
add(filepath.Join(home, ".local", "share"))
}
data_dirs := os.Getenv("XDG_DATA_DIRS")
if data_dirs == "" {
data_dirs = mime_cache_default_data_dirs
}
for _, x := range strings.Split(data_dirs, ":") {
add(x)
}
return
}
func load_mime_caches(paths []string) (ans mime_caches) {
for _, p := range paths {
c, err := load_mime_cache(p)
if err != nil {
if !errors.Is(err, fs.ErrNotExist) && !errors.Is(err, unix.ENOTDIR) {
fmt.Fprintln(os.Stderr, "Failed to load MIME cache file with error:", err)
}
continue
}
ans = append(ans, c)
}
return
}
var system_mime_caches = sync.OnceValue(func() mime_caches {
if runtime.GOOS == "darwin" {
return nil
}
return load_mime_caches(mime_cache_paths())
})
// HaveMimeCache returns true if at least one shared-mime-info mime.cache file
// was successfully loaded
func HaveMimeCache() bool {
return len(system_mime_caches()) > 0
}
// MimeTypeFromCache returns the MIME type for the specified file name, as
// determined by the shared-mime-info mime.cache files. Returns the empty
// string if no match is found. Only the base name of the file is used.
func MimeTypeFromCache(filename string) string {
if caches := system_mime_caches(); len(caches) > 0 {
return caches.lookup(filepath.Base(filename))
}
return ""
}
// MimeTypeAliases returns all names for the specified MIME type as known to
// the shared-mime-info mime.cache files, starting with the canonical name,
// followed by all aliases. Always contains at least the specified MIME type.
// The returned slice must not be modified.
func MimeTypeAliases(mime string) []string {
mime_type_aliases_cache.RLock()
ans, found := mime_type_aliases_cache.m[mime]
mime_type_aliases_cache.RUnlock()
if !found {
ans = system_mime_caches().aliases(mime)
mime_type_aliases_cache.Lock()
mime_type_aliases_cache.m[mime] = ans
mime_type_aliases_cache.Unlock()
}
return ans
}
var mime_type_aliases_cache = struct {
sync.RWMutex
m map[string][]string
}{m: make(map[string][]string)}