mirror of
https://github.com/kovidgoyal/kitty.git
synced 2026-11-18 22:22:22 +00:00
Recheck the path key before updating the cache. Include the inode so a replacement with the same size and modification time gets a different key. Version the key to invalidate previously cached previews. Retry once when a changed path still resolves to a regular file.
560 lines
15 KiB
Go
560 lines
15 KiB
Go
package disk_cache
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"runtime/debug"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
|
|
"github.com/google/go-cmp/cmp"
|
|
"github.com/kovidgoyal/kitty/tools/utils"
|
|
)
|
|
|
|
var _ = fmt.Print
|
|
|
|
func TestDiskCache(t *testing.T) {
|
|
tdir := t.TempDir()
|
|
dc, err := NewDiskCache(tdir, 64)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
dc2, err := NewDiskCache(tdir, dc.MaxSize)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ensure_entries := func() {
|
|
for _, x := range []*DiskCache{dc, dc2} {
|
|
if err = x.ensure_entries(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
}
|
|
arc := func(counts ...int) {
|
|
ensure_entries()
|
|
if diff := cmp.Diff(counts, []int{dc.read_count, dc2.read_count}); diff != "" {
|
|
t.Fatalf("disk cache has unexpected read count\n%s\n%s", diff, debug.Stack())
|
|
}
|
|
}
|
|
add := func(dc *DiskCache, key string, data map[string]string) {
|
|
d := make(map[string][]byte, len(data))
|
|
for k, v := range data {
|
|
d[k] = []byte(v)
|
|
}
|
|
if _, err := dc.Add(key, d); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ensure_entries()
|
|
}
|
|
|
|
m, err := dc.Get("missing", "one", "two")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(m) > 0 {
|
|
t.Fatalf("Unexpected return from missing: %s", m)
|
|
}
|
|
|
|
ad := func(key string, expected map[string]string) {
|
|
for _, x := range []*DiskCache{dc, dc2} {
|
|
actual, err := x.Get(key, utils.Keys(expected)...)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
for k, path := range actual {
|
|
d, err := os.ReadFile(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
actual[k] = string(d)
|
|
}
|
|
if diff := cmp.Diff(expected, actual); diff != "" {
|
|
t.Fatalf("Data for %s not equal: %s", key, diff)
|
|
}
|
|
ensure_entries()
|
|
}
|
|
}
|
|
ak := func(keys ...string) {
|
|
for i, x := range []*DiskCache{dc, dc2} {
|
|
kk, err := x.keys()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if diff := cmp.Diff(keys, kk); diff != "" {
|
|
t.Fatalf("wrong keys in %d: %s", i+1, diff)
|
|
}
|
|
}
|
|
ensure_entries()
|
|
}
|
|
add(dc, "k1", map[string]string{"1": "abcd", "2": "efgh"})
|
|
// dc2 reads the entries file dc created on construction, then re-reads it after dc.Add()
|
|
arc(0, 2)
|
|
ad("k1", map[string]string{"1": "abcd", "2": "efgh"})
|
|
arc(1, 3) // the two gets cause two updates
|
|
add(dc, "k1", map[string]string{"3": "ijk", "4": "lmo"})
|
|
arc(1, 4) // dc.Add() causes re-read in dc2
|
|
ak("k1")
|
|
arc(1, 4)
|
|
add(dc2, "k2", map[string]string{"1": "123456789"})
|
|
arc(2, 4) // dc2.Add() causes re-read in dc
|
|
ak("k1", "k2")
|
|
arc(2, 4)
|
|
ad("k1", map[string]string{"1": "abcd", "2": "efgh", "3": "ijk"})
|
|
if dc.entries.TotalSize != 14+9 {
|
|
t.Fatalf("TotalSize: %d != %d", dc.entries.TotalSize, 14+9)
|
|
}
|
|
arc(3, 5) // the two gets cause two updates
|
|
ak("k2", "k1")
|
|
dc.Get("k2")
|
|
arc(3, 6) // dc.Get() causes dc2 to read
|
|
ak("k1", "k2")
|
|
add(dc, "k3", map[string]string{"1": strings.Repeat("a", int(dc.MaxSize)-10)})
|
|
arc(3, 7) // dc.Add() causes dc2 to read
|
|
ak("k2", "k3")
|
|
// check that creating a new disk cache prunes
|
|
_, err = NewDiskCache(tdir, dc.MaxSize-8)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ak("k3")
|
|
arc(4, 8) // NewDiskCache()
|
|
|
|
// test the path api
|
|
path := filepath.Join(tdir, "source")
|
|
if err = os.WriteFile(path, []byte("abcdfjrof"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
key, _, err := dc.GetPath(path, "")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err = dc.AddPath(path, "", key, map[string][]byte{"1": []byte("1")}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_, _, entries_before, _ := dc.entries_from_folders()
|
|
if diff := cmp.Diff(1, len(dc.entries.PathMap)); diff != "" {
|
|
t.Fatalf("unexpected pathmap count: %s", diff)
|
|
}
|
|
if err = os.WriteFile(path, []byte("changed contents"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
new_key, _, err := dc.GetPath(path, "")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if new_key == key {
|
|
t.Fatal("expected a new key after changing the file")
|
|
}
|
|
if _, err = dc.AddPath(path, "", new_key, map[string][]byte{"1": []byte("2")}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if diff := cmp.Diff(1, len(dc.entries.PathMap)); diff != "" {
|
|
t.Fatalf("unexpected pathmap count: %s", diff)
|
|
}
|
|
_, _, entries_after, _ := dc.entries_from_folders()
|
|
if len(entries_before) != len(entries_after) {
|
|
t.Fatalf("unexpected entries: %s", entries_after)
|
|
}
|
|
arc(4, 9) // dc.AddPath() causes dc2 to read
|
|
}
|
|
|
|
func TestDiskCacheLock(t *testing.T) {
|
|
dc, err := NewDiskCache(t.TempDir(), 64)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err = dc.lock(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer dc.unlock()
|
|
if dc.lock_mutex.TryLock() {
|
|
dc.lock_mutex.Unlock()
|
|
t.Fatal("cache mutex released before the cache operation finished")
|
|
}
|
|
}
|
|
|
|
func TestDiskCacheConcurrentGet(t *testing.T) {
|
|
dc, err := NewDiskCache(t.TempDir(), 64)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err = dc.Add("sharedkey", map[string][]byte{"data": []byte("cached")}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var workers sync.WaitGroup
|
|
start := make(chan struct{})
|
|
for range 6 {
|
|
workers.Go(func() {
|
|
<-start
|
|
for range 8 {
|
|
paths, err := dc.Get("sharedkey", "data")
|
|
if err != nil {
|
|
t.Error(err)
|
|
return
|
|
}
|
|
if paths["data"] == "" {
|
|
t.Error("concurrent Get lost cached data")
|
|
return
|
|
}
|
|
if data, err := os.ReadFile(paths["data"]); err != nil || string(data) != "cached" {
|
|
t.Errorf("path returned by concurrent Get is not readable: %q, %v", data, err)
|
|
return
|
|
}
|
|
}
|
|
})
|
|
}
|
|
close(start)
|
|
workers.Wait()
|
|
}
|
|
|
|
func TestDiskCacheExportReplacesAtomically(t *testing.T) {
|
|
dc, err := NewDiskCache(t.TempDir(), 0)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
paths, err := dc.Add("key", map[string][]byte{"data": []byte("old")})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
exported := paths["data"]
|
|
// a Get of unchanged data must not touch the previously exported path
|
|
before, err := os.Stat(exported)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if paths, err = dc.Get("key", "data"); err != nil || paths["data"] != exported {
|
|
t.Fatalf("Get returned unexpected paths: %v, %v", paths, err)
|
|
}
|
|
if after, err := os.Stat(exported); err != nil || !os.SameFile(before, after) {
|
|
t.Fatalf("Get replaced an already exported file: %v", err)
|
|
}
|
|
// updated data must replace the exported path, leaving no temp files
|
|
if paths, err = dc.Add("key", map[string][]byte{"data": []byte("new")}); err != nil || paths["data"] != exported {
|
|
t.Fatalf("Add returned unexpected paths: %v, %v", paths, err)
|
|
}
|
|
if data, err := os.ReadFile(exported); err != nil || string(data) != "new" {
|
|
t.Fatalf("exported file not updated: %q, %v", data, err)
|
|
}
|
|
entries, err := os.ReadDir(dc.ResultsDir())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(entries) != 1 {
|
|
t.Fatalf("unexpected files in results dir: %v", entries)
|
|
}
|
|
}
|
|
|
|
func TestDiskCacheGetUntrackedKey(t *testing.T) {
|
|
dc, err := NewDiskCache(t.TempDir(), 64)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// simulate a key folder that is missing from entries.json
|
|
base := dc.folder_for_key("untracked")
|
|
if err = os.MkdirAll(base, 0o700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err = os.WriteFile(filepath.Join(base, "data"), []byte("orphan"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
paths, err := dc.Get("untracked", "data")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if data, err := os.ReadFile(paths["data"]); err != nil || string(data) != "orphan" {
|
|
t.Fatalf("could not read untracked key: %q, %v", data, err)
|
|
}
|
|
}
|
|
|
|
func TestDiskCacheLockErrors(t *testing.T) {
|
|
source := filepath.Join(t.TempDir(), "source")
|
|
if err := os.WriteFile(source, []byte("source"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, tc := range []struct {
|
|
name string
|
|
run func(*DiskCache) error
|
|
// whether run() locks dc itself, rather than some other DiskCache
|
|
uses_dc bool
|
|
}{
|
|
{"Get", func(dc *DiskCache) error {
|
|
_, err := dc.Get("key", "data")
|
|
return err
|
|
}, true},
|
|
{"GetPath", func(dc *DiskCache) error {
|
|
_, _, err := dc.GetPath(source, "", "data")
|
|
return err
|
|
}, true},
|
|
{"Remove", func(dc *DiskCache) error { return dc.Remove("key") }, true},
|
|
{"Clear", func(dc *DiskCache) error { return dc.Clear() }, true},
|
|
{"AddPath", func(dc *DiskCache) error {
|
|
_, err := dc.AddPath(source, "", "key", map[string][]byte{"data": []byte("updated")})
|
|
return err
|
|
}, true},
|
|
{"Add", func(dc *DiskCache) error {
|
|
_, err := dc.Add("key", map[string][]byte{"data": []byte("updated")})
|
|
return err
|
|
}, true},
|
|
{"NewDiskCache", func(dc *DiskCache) error {
|
|
_, err := NewDiskCache(dc.Path, dc.MaxSize)
|
|
return err
|
|
}, false},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
dc, err := NewDiskCache(t.TempDir(), 64)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err = dc.Add("key", map[string][]byte{"data": []byte("cached")}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
lock_path := filepath.Join(dc.Path, "lockfile")
|
|
if err = os.Remove(lock_path); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err = os.Mkdir(lock_path, 0o700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err = tc.run(dc); err == nil {
|
|
t.Error("cache operation ignored failure to acquire the lock")
|
|
}
|
|
if tc.uses_dc {
|
|
if !dc.lock_mutex.TryLock() {
|
|
t.Fatal("failed lock attempt did not release the mutex")
|
|
}
|
|
dc.lock_mutex.Unlock()
|
|
if dc.lock_file != nil {
|
|
t.Fatal("failed lock attempt did not close the lock file")
|
|
}
|
|
}
|
|
if err = os.Remove(lock_path); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
paths, err := dc.Get("key", "data")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
data, err := os.ReadFile(paths["data"])
|
|
if err != nil || string(data) != "cached" {
|
|
t.Fatalf("cache data changed after a failed lock attempt: %q, %v", data, err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestDiskCachePathExtraKey(t *testing.T) {
|
|
tdir := t.TempDir()
|
|
dc, err := NewDiskCache(filepath.Join(tdir, "cache"), 1024*1024)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
path := filepath.Join(tdir, "source")
|
|
if err = os.WriteFile(path, []byte("contents"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
keys := map[string]string{}
|
|
for _, extra := range []string{"", "a", "b"} {
|
|
key, _, err := dc.GetPath(path, extra)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if prev, found := keys[key]; found {
|
|
t.Fatalf("extra keys %q and %q gave the same key", prev, extra)
|
|
}
|
|
keys[key] = extra
|
|
again, _, err := dc.GetPath(path, extra)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if again != key {
|
|
t.Fatalf("extra key %q did not give a stable key", extra)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestDiskCacheRejectsStalePathKey(t *testing.T) {
|
|
tdir := t.TempDir()
|
|
dc, err := NewDiskCache(filepath.Join(tdir, "cache"), 1024*1024)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
path := filepath.Join(tdir, "source")
|
|
if err = os.WriteFile(path, []byte("original"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
key, _, err := dc.GetPath(path, "")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
info, err := os.Stat(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
replacement := filepath.Join(tdir, "replacement")
|
|
if err = os.WriteFile(replacement, []byte("replaced"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err = os.Chtimes(replacement, info.ModTime(), info.ModTime()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err = os.Rename(replacement, path); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err = dc.AddPath(path, "", key, map[string][]byte{"data": []byte("preview")}); err == nil {
|
|
t.Fatal("AddPath accepted a key for the replaced file")
|
|
}
|
|
if ans, err := dc.Get(key); err != nil || len(ans) != 0 {
|
|
t.Fatalf("stale key should have no cached items, got %v, %v", ans, err)
|
|
}
|
|
}
|
|
|
|
func TestDiskCacheSharedPathEntries(t *testing.T) {
|
|
tdir := t.TempDir()
|
|
dc, err := NewDiskCache(filepath.Join(tdir, "cache"), 1024*1024)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
target, other := filepath.Join(tdir, "target"), filepath.Join(tdir, "other")
|
|
for _, x := range []string{target, other} {
|
|
if err = os.WriteFile(x, []byte(filepath.Base(x)), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
one, two := filepath.Join(tdir, "one"), filepath.Join(tdir, "two")
|
|
for _, x := range []string{one, two} {
|
|
if err = os.Symlink(target, x); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
// get returns the key for path and whether it was a cache hit, adding an
|
|
// entry for path on a miss.
|
|
get := func(path string) (string, bool) {
|
|
t.Helper()
|
|
key, ans, err := dc.GetPath(path, "")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(ans) > 0 {
|
|
return key, true
|
|
}
|
|
if _, err = dc.AddPath(path, "", key, map[string][]byte{"data": []byte(path)}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return key, false
|
|
}
|
|
shared, _ := get(one)
|
|
if key, hit := get(two); !hit || key != shared {
|
|
t.Fatal("symlinks to the same file did not share a cache entry")
|
|
}
|
|
retarget := func(link string) {
|
|
t.Helper()
|
|
if err = os.Remove(link); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err = os.Symlink(other, link); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
retarget(one)
|
|
if _, hit := get(one); hit {
|
|
t.Fatal("retargeted symlink got a cache hit")
|
|
}
|
|
if _, hit := get(two); !hit {
|
|
t.Fatal("retargeting one symlink removed the entry still used by another")
|
|
}
|
|
retarget(two)
|
|
if _, hit := get(two); !hit {
|
|
t.Fatal("symlinks to the same file did not share a cache entry")
|
|
}
|
|
if ans, err := dc.Get(shared); err != nil {
|
|
t.Fatal(err)
|
|
} else if len(ans) > 0 {
|
|
t.Fatal("entry no longer used by any path was not removed")
|
|
}
|
|
}
|
|
|
|
// The first path added to a new cache must be recorded, so that its entry is
|
|
// removed when the file changes.
|
|
func TestDiskCacheFirstPathRecorded(t *testing.T) {
|
|
tdir := t.TempDir()
|
|
dc, err := NewDiskCache(filepath.Join(tdir, "cache"), 1024*1024)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
path := filepath.Join(tdir, "source")
|
|
var keys []string
|
|
for _, data := range []string{"first", "second version"} {
|
|
if err = os.WriteFile(path, []byte(data), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
key, _, err := dc.GetPath(path, "")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err = dc.AddPath(path, "", key, map[string][]byte{"data": []byte(data)}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
keys = append(keys, key)
|
|
}
|
|
if ans, err := dc.Get(keys[0]); err != nil {
|
|
t.Fatal(err)
|
|
} else if len(ans) > 0 {
|
|
t.Fatal("entry for the previous version of the file was not removed")
|
|
}
|
|
}
|
|
|
|
// Failing to remove an entry no longer used must not turn a cache hit into an error.
|
|
func TestDiskCacheHitWithFailedCleanup(t *testing.T) {
|
|
if os.Geteuid() == 0 {
|
|
t.Skip("permissions are not enforced for root")
|
|
}
|
|
tdir := t.TempDir()
|
|
dc, err := NewDiskCache(filepath.Join(tdir, "cache"), 1024*1024)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
target, other, link := filepath.Join(tdir, "target"), filepath.Join(tdir, "other"), filepath.Join(tdir, "link")
|
|
for _, x := range []string{target, other} {
|
|
if err = os.WriteFile(x, []byte(filepath.Base(x)), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if err = os.Symlink(target, link); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
add := func(path string) string {
|
|
t.Helper()
|
|
key, _, err := dc.GetPath(path, "")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err = dc.AddPath(path, "", key, map[string][]byte{"data": []byte(path)}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return key
|
|
}
|
|
orphan := add(link)
|
|
add(other)
|
|
if err = os.Remove(link); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err = os.Symlink(other, link); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
parent := filepath.Dir(dc.folder_for_key(orphan))
|
|
if err = os.Chmod(parent, 0o500); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { _ = os.Chmod(parent, 0o700) })
|
|
_, ans, err := dc.GetPath(link, "")
|
|
if err != nil {
|
|
t.Fatalf("cache hit failed because of a failed cleanup: %s", err)
|
|
}
|
|
if len(ans) == 0 {
|
|
t.Fatal("expected a cache hit for the retargeted symlink")
|
|
}
|
|
}
|