From: Diogo Franco (Kovensky) Date: Tue, 16 Jul 2013 21:15:57 +0000 (-0300) Subject: Initial commit X-Git-Tag: v0.0.1~8 X-Git-Url: https://git.lizzy.rs/?a=commitdiff_plain;h=2b4c72505687ba3a92833b40cb88cc695c93c361;p=go-fscache.git Initial commit Cache that uses the filesystem as its keys. --- 2b4c72505687ba3a92833b40cb88cc695c93c361 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3e74654 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +[_.]testdir diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..4d65661 --- /dev/null +++ b/LICENSE @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2013 Diogo Franco + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..fa864a7 --- /dev/null +++ b/README.md @@ -0,0 +1,4 @@ +go-fscache +========== + +Cache that uses the filesystem as its keys diff --git a/cache_posix.go b/cache_posix.go new file mode 100755 index 0000000..6e0b104 --- /dev/null +++ b/cache_posix.go @@ -0,0 +1,15 @@ +// +build !windows + +package fscache + +import ( + "strings" +) + +func filterDots(parts ...string) []string { + return parts +} + +func filterDotsAll(parts ...string) []string { + return parts +} diff --git a/cache_windows.go b/cache_windows.go new file mode 100755 index 0000000..779e0f1 --- /dev/null +++ b/cache_windows.go @@ -0,0 +1,23 @@ +package fscache + +import ( + "strings" +) + +func filterDots(parts ...string) []string { + // no subdir + if len(parts) < 2 { + return parts + } + filterDotsAll(parts[:len(parts)-1]...) + return parts +} + +func filterDotsAll(parts ...string) []string { + for i := range parts { + if parts[i][0] == '.' { + parts[i] = strings.Replace(parts[i], ".", "_", 1) + } + } + return parts +} diff --git a/cachedir.go b/cachedir.go new file mode 100755 index 0000000..0873669 --- /dev/null +++ b/cachedir.go @@ -0,0 +1,83 @@ +package fscache + +import ( + "os" + "path/filepath" + "sync" +) + +type CacheDir struct { + mutex sync.RWMutex + + compressionLevel int + cacheDir string +} + +// Creates (or opens) a CacheDir using the given path. +func NewCacheDir(path string) (cd *CacheDir, err error) { + cd = &CacheDir{ + compressionLevel: DefaultCompressionLevel, + } + if err = cd.SetCacheDir(path); err != nil { + return nil, err + } + return +} + +// Sets the directory that will back this cache. +// +// Will try to os.MkdirAll the given path; if that fails, +// then the CacheDir is not modified. +func (cd *CacheDir) SetCacheDir(path string) (err error) { + cd.mutex.Lock() + defer cd.mutex.Unlock() + + path = filepath.Join(filterDotsAll(filepath.SplitList(path)...)...) + + if err = os.MkdirAll(path, 0777); err != nil { + return + } + cd.cacheDir = path + return +} + +// Gets the path to the cache directory. +func (cd *CacheDir) GetCacheDir() string { + cd.mutex.RLock() + defer cd.mutex.RUnlock() + + return cd.cacheDir +} + +// Opens the file that backs the specified key. +func (cd *CacheDir) Open(key ...CacheKey) (fh *os.File, err error) { + return os.Open(cd.cachePath(key...)) +} + +// Opens the file that backs the specified key using os.OpenFile. +// +// The permission bits are always 0666, which then get filtered by umask. +func (cd *CacheDir) OpenFlags(flags int, key ...CacheKey) (fh *os.File, err error) { + return os.OpenFile(cd.cachePath(key...), flags, 0666) +} + +// Creates a new file to back the specified key. +func (cd *CacheDir) Create(key ...CacheKey) (fh *os.File, err error) { + subItem := cd.cachePath(key...) + subDir := filepath.Dir(subItem) + + if err = os.MkdirAll(subDir, 0777); err != nil { + return nil, err + } + return os.Create(subItem) +} + +// Deletes the file that backs the specified key. +func (cd *CacheDir) Delete(key ...CacheKey) (err error) { + return os.Remove(cd.cachePath(key...)) +} + +// Deletes the specified key and all subkeys. +func (cd *CacheDir) DeleteAll(key ...CacheKey) (err error) { + return os.RemoveAll(cd.cachePath(key...)) +} diff --git a/cachegob.go b/cachegob.go new file mode 100755 index 0000000..34024be --- /dev/null +++ b/cachegob.go @@ -0,0 +1,178 @@ +package fscache + +import ( + "bytes" + "compress/gzip" + "encoding/gob" + "errors" + "fmt" + "io" + "os" + "reflect" + "time" +) + +// The default compression level of new CacheDir objects. +const DefaultCompressionLevel = gzip.BestCompression + +func (cd *CacheDir) SetCompressionLevel(level int) { + cd.mutex.Lock() + defer cd.mutex.Unlock() + + cd.compressionLevel = level +} + +// Retrieves the current gzip compression level. +func (cd *CacheDir) GetCompressionLevel() int { + cd.mutex.Lock() + defer cd.mutex.Unlock() + + return cd.compressionLevel +} + +// Calls Get to retrieve the requested key from the cache. +// +// If the key is expired, then it is removed from the cache. +func (cd *CacheDir) GetAndExpire(v interface{}, max time.Duration, key ...CacheKey) (mtime time.Time, expired bool, err error) { + mtime, err = cd.Get(v, key...) + + if err != nil && time.Now().Sub(mtime) > max { + expired = true + err = cd.Delete(key...) + } + return +} + +// Gets the requested key from the cache. The given interface{} must be a pointer +// or otherwise be modifiable; otherwise Get will panic. +func (cd *CacheDir) Get(v interface{}, key ...CacheKey) (mtime time.Time, err error) { + val := reflect.ValueOf(v) + if k := val.Kind(); k == reflect.Ptr || k == reflect.Interface { + val = val.Elem() + } + if !val.CanSet() { + // API caller error + panic("(*cacheDir).Get(): given Cacheable is not setable") + } + + lock, err := cd.Lock(key...) + if err != nil { + return + } + defer func() { + // We may unlock it early + if lock != nil { + lock.Unlock() + } + }() + + fh, err := cd.Open(key...) + if err != nil { + return + } + stat, err := fh.Stat() + if err != nil { + return + } + mtime = stat.ModTime() + + buf := bytes.Buffer{} + if _, err = io.Copy(&buf, fh); err != nil { + fh.Close() + return + } + if err = fh.Close(); err != nil { + return + } + + if lock != nil { + // early unlock + lock.Unlock() + lock = nil + } + + gz, err := gzip.NewReader(&buf) + if err != nil { + return + } + defer func() { + if e := gz.Close(); err == nil { + err = e + } + }() + + switch f := gz.Header.Comment; f { + case "encoding/gob": + dec := gob.NewDecoder(gz) + err = dec.Decode(v) + default: + err = errors.New(fmt.Sprintf("Cached data (format %q) is not in a known format", f)) + } + + return +} + +// Stores the given interface{} in the cache. Returns the size of the resulting file and the error, if any. +// +// Compresses the resulting data using gzip with the compression level set by SetCompressionLevel(). +func (cd *CacheDir) Set(v interface{}, key ...CacheKey) (n int64, err error) { + if v := reflect.ValueOf(v); !v.IsValid() { + panic("reflect.ValueOf() returned invaled value") + } else if k := v.Kind(); k == reflect.Ptr || k == reflect.Interface { + if v.IsNil() { + return // no point in saving nil + } + } + + // First we encode to memory -- we don't want to create/truncate a file and put bad data in it. + buf := bytes.Buffer{} + gz, err := gzip.NewWriterLevel(&buf, gzip.BestCompression) + if err != nil { + return 0, err + } + gz.Header.Comment = "encoding/gob" + + enc := gob.NewEncoder(gz) + err = enc.Encode(v) + + if e := gz.Close(); err == nil { + err = e + } + + if err != nil { + return 0, err + } + + // We have good data, time to actually put it in the cache + lock, err := cd.Lock(key...) + switch { + case err == nil: + // AOK + defer lock.Unlock() + case os.IsNotExist(err): + // new file + default: + return 0, err + } + + fh, err := cd.Create(key...) + if err != nil { + return 0, err + } + if lock == nil { + // the file didn't exist before, but it does now + lock, err = cd.Lock(key...) + if err != nil { + return 0, err + } + defer lock.Unlock() + } + + defer func() { + if e := fh.Close(); err == nil { + err = e + } + }() + n, err = io.Copy(fh, &buf) + return +} diff --git a/cachegob_test.go b/cachegob_test.go new file mode 100755 index 0000000..714aba3 --- /dev/null +++ b/cachegob_test.go @@ -0,0 +1,39 @@ +package fscache_test + +import ( + "github.com/Kovensky/go-fscache" + "math/rand" + "testing" + "time" +) + +func init() { + rand.Seed(time.Now().UnixNano()) +} + +func TestCache_Set(T *testing.T) { + cd, err := fscache.NewCacheDir(".testdir") + if err != nil { + T.Fatal(err) + return + } + + val := rand.Int63() + val2 := ^val // make it different + + _, err = cd.Set(val, "test", "set") + if err != nil { + T.Fatal(err) + return + } + + _, err = cd.Get(&val2, "test", "set") + if err != nil { + T.Fatal(err) + return + } + + if val != val2 { + T.Errorf("Expected %d, got %d", val, val2) + } +} diff --git a/cachekey.go b/cachekey.go new file mode 100755 index 0000000..3c489ca --- /dev/null +++ b/cachekey.go @@ -0,0 +1,88 @@ +package fscache + +import ( + "fmt" + "path/filepath" + "regexp" + "time" +) + +// An arbitrary object that can be stringified by fmt.Sprint(). +// +// The stringification is filtered to ensure it doesn't contain characters +// that are invalid on Windows, which has the most restrictive filesystem. +// The "bad" characters (\, /, :, *, ?, ", <, >, |) are replaced with _. +// +// On a list of CacheKeys, the last component is taken to represent a file +// and all the other components represent the intermediary directories. +// This means that it's not possible to have subkeys of an existing file key. +type CacheKey interface{} + +// All "bad characters" that can't go in Windows paths. +// It's a superset of the "bad characters" on other OSes, so this works. +var badPath = regexp.MustCompile(`[\\/:\*\?\"<>\|]`) + +func stringify(stuff ...CacheKey) []string { + ret := make([]string, len(stuff)) + for i := range stuff { + s := fmt.Sprint(stuff[i]) + ret[i] = badPath.ReplaceAllLiteralString(s, "_") + } + return ret +} + +// Each key but the last is treated as a directory. +// The last key is treated as a regular file. +// +// This also means that cache keys that are file-backed +// cannot have subkeys. +func (cd *CacheDir) cachePath(key ...CacheKey) string { + parts := append([]string{cd.GetCacheDir()}, stringify(key...)...) + p := filepath.Join(filterDots(parts...)...) + return p +} + +var invalidPath = []CacheKey{".invalid"} + +// Returns the time the given key was marked as invalid. +// If the key is valid, then calling IsZero() on the returned +// time will return true. +func (cd *CacheDir) GetInvalid(key ...CacheKey) (ts time.Time) { + invKey := append(invalidPath, key...) + + stat, _ := cd.Stat(invKey...) + return stat.ModTime() +} + +// Checks if the given key is not marked as invalid, or if it is, +// checks if it was marked more than maxDuration time ago. +// +// Calls UnsetInvalid if the keys are valid. +func (cd *CacheDir) IsValid(maxDuration time.Duration, key ...CacheKey) bool { + ts := cd.GetInvalid(key...) + + switch { + case ts.IsZero(): + return true + case time.Now().Sub(ts) > maxDuration: + cd.UnsetInvalid(key...) + return true + default: + return false + } +} + +// Deletes the given key and caches it as invalid. +func (cd *CacheDir) SetInvalid(key ...CacheKey) error { + invKey := append(invalidPath, key...) + + cd.Delete(key...) + return cd.Touch(invKey...) +} + +// Removes the given key from the invalid key cache. +func (cd *CacheDir) UnsetInvalid(key ...CacheKey) error { + invKey := append(invalidPath, key...) + + return cd.Delete(invKey...) +} diff --git a/cachelock.go b/cachelock.go new file mode 100755 index 0000000..60861fc --- /dev/null +++ b/cachelock.go @@ -0,0 +1,16 @@ +package fscache + +import ( + "github.com/Kovensky/go-fscache/lock" +) + +// Locks the file that backs the given key. +// +// If the call is successful, it's the caller's responsibility to call Unlock on the returned lock. +func (cd *CacheDir) Lock(key ...CacheKey) (lock.FileLock, error) { + l, err := lock.LockFile(cd.cachePath(key...)) + if l != nil { + l.Lock() + } + return l, err +} diff --git a/cachestat.go b/cachestat.go new file mode 100755 index 0000000..a714829 --- /dev/null +++ b/cachestat.go @@ -0,0 +1,57 @@ +package fscache + +import ( + "os" + "time" +) + +// Calls os.Stat() on the file or folder that backs the given key. +func (cd *CacheDir) Stat(key ...CacheKey) (stat os.FileInfo, err error) { + lock, err := cd.Lock(key...) + if err != nil { + return + } + defer lock.Unlock() + + fh, err := cd.Open(key...) + if err != nil { + return + } + defer fh.Close() + + return fh.Stat() +} + +// Updates the mtime of the file backing the given key. +// +// Creates an empty file if it doesn't exist. +func (cd *CacheDir) Touch(key ...CacheKey) (err error) { + lock, err := cd.Lock(key...) + switch { + case err == nil: + // AOK + defer lock.Unlock() + case os.IsNotExist(err): + // new file + default: + return + } + + if err = os.Chtimes(cd.cachePath(key...), time.Now(), time.Now()); err == nil { + return + } + + fh, err := cd.OpenFlags(os.O_APPEND|os.O_CREATE, key...) + switch { + case err == nil: + // AOK + case os.IsNotExist(err): // directory path not created yet + fh, err = cd.Create(key...) + if err != nil { + return + } + default: + return + } + return fh.Close() +} diff --git a/cachestat_test.go b/cachestat_test.go new file mode 100755 index 0000000..735072c --- /dev/null +++ b/cachestat_test.go @@ -0,0 +1,97 @@ +package fscache_test + +import ( + "github.com/Kovensky/go-fscache" + "math/rand" + "testing" + "time" +) + +func init() { + rand.Seed(time.Now().UnixNano()) +} + +func TestCache_Stat(T *testing.T) { + cd, err := fscache.NewCacheDir(".testdir") + if err != nil { + T.Fatal(err) + return + } + + err = cd.Touch("test", "stat") + if err != nil { + T.Fatal(err) + return + } + + stat, err := cd.Stat("test", "stat") + if err != nil { + T.Fatal(err) + return + } + + T.Log("IsDir:", stat.IsDir()) + T.Log("ModTime:", stat.ModTime()) + T.Log("Name: " + stat.Name()) + T.Log("Size:", stat.Size(), "bytes") + + if stat.IsDir() || stat.ModTime().IsZero() || stat.Name() != "stat" || stat.Size() != 0 { + T.Error("Stat returned unexpected data") + } +} + +func TestCache_Touch(T *testing.T) { + cd, err := fscache.NewCacheDir(".testdir") + if err != nil { + T.Fatal(err) + return + } + + err = cd.Touch("test", "touch", "file") + if err != nil { + T.Fatal(err) + return + } + + stat, err := cd.Stat("test", "touch") + if err != nil { + T.Fatal(err) + return + } + + if !stat.IsDir() { + T.Error("Expected touch to be a dir, file found") + } + + // can be anything -- just make a nonblank file + size, err := cd.Set(stat.ModTime(), "test", "touch", "subdir", "file") + if err != nil { + T.Fatal(err) + return + } + + stat, err = cd.Stat("test", "touch", "subdir", "file") + if err != nil { + T.Fatal(err) + return + } + if stat.Size() != size { + panic("File modified outside of test") + } + + T.Log("Waiting 5ms") + time.Sleep(5 * time.Millisecond) + + err = cd.Touch("test", "touch", "subdir", "file") + if err != nil { + T.Fatal(err) + } + stat2, err := cd.Stat("test", "touch", "subdir", "file") + + if stat.Size() != stat2.Size() { + T.Errorf("Touch modified the size") + } + if !stat2.ModTime().After(stat.ModTime()) { + T.Errorf("Touch did not update timestamp (FAT filesystem?)") + } +} diff --git a/fscache.go b/fscache.go new file mode 100755 index 0000000..f02b617 --- /dev/null +++ b/fscache.go @@ -0,0 +1,6 @@ +// Cache that uses the filesystem for indexing keys. +// +// Prefer running on case-sensitive filesystems; running on a +// case-insensitive one has the side-effect that keys are also +// case-insensitive. +package fscache diff --git a/lock/lock.go b/lock/lock.go new file mode 100644 index 0000000..80cff30 --- /dev/null +++ b/lock/lock.go @@ -0,0 +1,13 @@ +// Wrapper for github.com/tgulacsi/go-locking since it doesn't compile on windows. +// +// On windows, returns a dummy lock that always succeeds. On other OSes, +// returns a *locking.FLock. +// +// Windows also does file locking on its own, but with different +// semantics. +package lock + +type FileLock interface { + Lock() error + Unlock() error +} diff --git a/lock/lock_posix.go b/lock/lock_posix.go new file mode 100644 index 0000000..3d7a77d --- /dev/null +++ b/lock/lock_posix.go @@ -0,0 +1,25 @@ +// +build !windows + +package lock + +import "github.com/tgulacsi/go-locking" + +type flockLock struct { + locking.FLock +} + +func LockFile(p string) (FileLock, error) { + flock, err := locking.NewFLock(p) + if err == nil { + return &flockLock{FLock: flock}, nil + } + return nil, err +} + +func (fl *flockLock) Lock() error { + return fl.FLock.Lock() +} + +func (fl *flockLock) Unlock() error { + return fl.FLock.Unlock() +} diff --git a/lock/lock_windows.go b/lock/lock_windows.go new file mode 100644 index 0000000..6b897ea --- /dev/null +++ b/lock/lock_windows.go @@ -0,0 +1,10 @@ +package lock + +func LockFile(p string) (FileLock, error) { + return &winLock{}, nil +} + +type winLock struct{} + +func (_ *winLock) Lock() error { return nil } +func (_ *winLock) Unlock() error { return nil }