Cache that uses the filesystem as its keys.
--- /dev/null
+[_.]testdir
--- /dev/null
+The MIT License (MIT)
+
+Copyright (c) 2013 Diogo Franco
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
--- /dev/null
+go-fscache
+==========
+
+Cache that uses the filesystem as its keys
--- /dev/null
+// +build !windows
+
+package fscache
+
+import (
+ "strings"
+)
+
+func filterDots(parts ...string) []string {
+ return parts
+}
+
+func filterDotsAll(parts ...string) []string {
+ return parts
+}
--- /dev/null
+package fscache
+
+import (
+ "strings"
+)
+
+func filterDots(parts ...string) []string {
+ // no subdir
+ if len(parts) < 2 {
+ return parts
+ }
+ filterDotsAll(parts[:len(parts)-1]...)
+ return parts
+}
+
+func filterDotsAll(parts ...string) []string {
+ for i := range parts {
+ if parts[i][0] == '.' {
+ parts[i] = strings.Replace(parts[i], ".", "_", 1)
+ }
+ }
+ return parts
+}
--- /dev/null
+package fscache
+
+import (
+ "os"
+ "path/filepath"
+ "sync"
+)
+
+type CacheDir struct {
+ mutex sync.RWMutex
+
+ compressionLevel int
+ cacheDir string
+}
+
+// Creates (or opens) a CacheDir using the given path.
+func NewCacheDir(path string) (cd *CacheDir, err error) {
+ cd = &CacheDir{
+ compressionLevel: DefaultCompressionLevel,
+ }
+ if err = cd.SetCacheDir(path); err != nil {
+ return nil, err
+ }
+ return
+}
+
+// Sets the directory that will back this cache.
+//
+// Will try to os.MkdirAll the given path; if that fails,
+// then the CacheDir is not modified.
+func (cd *CacheDir) SetCacheDir(path string) (err error) {
+ cd.mutex.Lock()
+ defer cd.mutex.Unlock()
+
+ path = filepath.Join(filterDotsAll(filepath.SplitList(path)...)...)
+
+ if err = os.MkdirAll(path, 0777); err != nil {
+ return
+ }
+ cd.cacheDir = path
+ return
+}
+
+// Gets the path to the cache directory.
+func (cd *CacheDir) GetCacheDir() string {
+ cd.mutex.RLock()
+ defer cd.mutex.RUnlock()
+
+ return cd.cacheDir
+}
+
+// Opens the file that backs the specified key.
+func (cd *CacheDir) Open(key ...CacheKey) (fh *os.File, err error) {
+ return os.Open(cd.cachePath(key...))
+}
+
+// Opens the file that backs the specified key using os.OpenFile.
+//
+// The permission bits are always 0666, which then get filtered by umask.
+func (cd *CacheDir) OpenFlags(flags int, key ...CacheKey) (fh *os.File, err error) {
+ return os.OpenFile(cd.cachePath(key...), flags, 0666)
+}
+
+// Creates a new file to back the specified key.
+func (cd *CacheDir) Create(key ...CacheKey) (fh *os.File, err error) {
+ subItem := cd.cachePath(key...)
+ subDir := filepath.Dir(subItem)
+
+ if err = os.MkdirAll(subDir, 0777); err != nil {
+ return nil, err
+ }
+ return os.Create(subItem)
+}
+
+// Deletes the file that backs the specified key.
+func (cd *CacheDir) Delete(key ...CacheKey) (err error) {
+ return os.Remove(cd.cachePath(key...))
+}
+
+// Deletes the specified key and all subkeys.
+func (cd *CacheDir) DeleteAll(key ...CacheKey) (err error) {
+ return os.RemoveAll(cd.cachePath(key...))
+}
--- /dev/null
+package fscache
+
+import (
+ "bytes"
+ "compress/gzip"
+ "encoding/gob"
+ "errors"
+ "fmt"
+ "io"
+ "os"
+ "reflect"
+ "time"
+)
+
+// The default compression level of new CacheDir objects.
+const DefaultCompressionLevel = gzip.BestCompression
+
+func (cd *CacheDir) SetCompressionLevel(level int) {
+ cd.mutex.Lock()
+ defer cd.mutex.Unlock()
+
+ cd.compressionLevel = level
+}
+
+// Retrieves the current gzip compression level.
+func (cd *CacheDir) GetCompressionLevel() int {
+ cd.mutex.Lock()
+ defer cd.mutex.Unlock()
+
+ return cd.compressionLevel
+}
+
+// Calls Get to retrieve the requested key from the cache.
+//
+// If the key is expired, then it is removed from the cache.
+func (cd *CacheDir) GetAndExpire(v interface{}, max time.Duration, key ...CacheKey) (mtime time.Time, expired bool, err error) {
+ mtime, err = cd.Get(v, key...)
+
+ if err != nil && time.Now().Sub(mtime) > max {
+ expired = true
+ err = cd.Delete(key...)
+ }
+ return
+}
+
+// Gets the requested key from the cache. The given interface{} must be a pointer
+// or otherwise be modifiable; otherwise Get will panic.
+func (cd *CacheDir) Get(v interface{}, key ...CacheKey) (mtime time.Time, err error) {
+ val := reflect.ValueOf(v)
+ if k := val.Kind(); k == reflect.Ptr || k == reflect.Interface {
+ val = val.Elem()
+ }
+ if !val.CanSet() {
+ // API caller error
+ panic("(*cacheDir).Get(): given Cacheable is not setable")
+ }
+
+ lock, err := cd.Lock(key...)
+ if err != nil {
+ return
+ }
+ defer func() {
+ // We may unlock it early
+ if lock != nil {
+ lock.Unlock()
+ }
+ }()
+
+ fh, err := cd.Open(key...)
+ if err != nil {
+ return
+ }
+ stat, err := fh.Stat()
+ if err != nil {
+ return
+ }
+ mtime = stat.ModTime()
+
+ buf := bytes.Buffer{}
+ if _, err = io.Copy(&buf, fh); err != nil {
+ fh.Close()
+ return
+ }
+ if err = fh.Close(); err != nil {
+ return
+ }
+
+ if lock != nil {
+ // early unlock
+ lock.Unlock()
+ lock = nil
+ }
+
+ gz, err := gzip.NewReader(&buf)
+ if err != nil {
+ return
+ }
+ defer func() {
+ if e := gz.Close(); err == nil {
+ err = e
+ }
+ }()
+
+ switch f := gz.Header.Comment; f {
+ case "encoding/gob":
+ dec := gob.NewDecoder(gz)
+ err = dec.Decode(v)
+ default:
+ err = errors.New(fmt.Sprintf("Cached data (format %q) is not in a known format", f))
+ }
+
+ return
+}
+
+// Stores the given interface{} in the cache. Returns the size of the resulting file and the error, if any.
+//
+// Compresses the resulting data using gzip with the compression level set by SetCompressionLevel().
+func (cd *CacheDir) Set(v interface{}, key ...CacheKey) (n int64, err error) {
+ if v := reflect.ValueOf(v); !v.IsValid() {
+ panic("reflect.ValueOf() returned invaled value")
+ } else if k := v.Kind(); k == reflect.Ptr || k == reflect.Interface {
+ if v.IsNil() {
+ return // no point in saving nil
+ }
+ }
+
+ // First we encode to memory -- we don't want to create/truncate a file and put bad data in it.
+ buf := bytes.Buffer{}
+ gz, err := gzip.NewWriterLevel(&buf, gzip.BestCompression)
+ if err != nil {
+ return 0, err
+ }
+ gz.Header.Comment = "encoding/gob"
+
+ enc := gob.NewEncoder(gz)
+ err = enc.Encode(v)
+
+ if e := gz.Close(); err == nil {
+ err = e
+ }
+
+ if err != nil {
+ return 0, err
+ }
+
+ // We have good data, time to actually put it in the cache
+ lock, err := cd.Lock(key...)
+ switch {
+ case err == nil:
+ // AOK
+ defer lock.Unlock()
+ case os.IsNotExist(err):
+ // new file
+ default:
+ return 0, err
+ }
+
+ fh, err := cd.Create(key...)
+ if err != nil {
+ return 0, err
+ }
+ if lock == nil {
+ // the file didn't exist before, but it does now
+ lock, err = cd.Lock(key...)
+ if err != nil {
+ return 0, err
+ }
+ defer lock.Unlock()
+ }
+
+ defer func() {
+ if e := fh.Close(); err == nil {
+ err = e
+ }
+ }()
+ n, err = io.Copy(fh, &buf)
+ return
+}
--- /dev/null
+package fscache_test
+
+import (
+ "github.com/Kovensky/go-fscache"
+ "math/rand"
+ "testing"
+ "time"
+)
+
+func init() {
+ rand.Seed(time.Now().UnixNano())
+}
+
+func TestCache_Set(T *testing.T) {
+ cd, err := fscache.NewCacheDir(".testdir")
+ if err != nil {
+ T.Fatal(err)
+ return
+ }
+
+ val := rand.Int63()
+ val2 := ^val // make it different
+
+ _, err = cd.Set(val, "test", "set")
+ if err != nil {
+ T.Fatal(err)
+ return
+ }
+
+ _, err = cd.Get(&val2, "test", "set")
+ if err != nil {
+ T.Fatal(err)
+ return
+ }
+
+ if val != val2 {
+ T.Errorf("Expected %d, got %d", val, val2)
+ }
+}
--- /dev/null
+package fscache
+
+import (
+ "fmt"
+ "path/filepath"
+ "regexp"
+ "time"
+)
+
+// An arbitrary object that can be stringified by fmt.Sprint().
+//
+// The stringification is filtered to ensure it doesn't contain characters
+// that are invalid on Windows, which has the most restrictive filesystem.
+// The "bad" characters (\, /, :, *, ?, ", <, >, |) are replaced with _.
+//
+// On a list of CacheKeys, the last component is taken to represent a file
+// and all the other components represent the intermediary directories.
+// This means that it's not possible to have subkeys of an existing file key.
+type CacheKey interface{}
+
+// All "bad characters" that can't go in Windows paths.
+// It's a superset of the "bad characters" on other OSes, so this works.
+var badPath = regexp.MustCompile(`[\\/:\*\?\"<>\|]`)
+
+func stringify(stuff ...CacheKey) []string {
+ ret := make([]string, len(stuff))
+ for i := range stuff {
+ s := fmt.Sprint(stuff[i])
+ ret[i] = badPath.ReplaceAllLiteralString(s, "_")
+ }
+ return ret
+}
+
+// Each key but the last is treated as a directory.
+// The last key is treated as a regular file.
+//
+// This also means that cache keys that are file-backed
+// cannot have subkeys.
+func (cd *CacheDir) cachePath(key ...CacheKey) string {
+ parts := append([]string{cd.GetCacheDir()}, stringify(key...)...)
+ p := filepath.Join(filterDots(parts...)...)
+ return p
+}
+
+var invalidPath = []CacheKey{".invalid"}
+
+// Returns the time the given key was marked as invalid.
+// If the key is valid, then calling IsZero() on the returned
+// time will return true.
+func (cd *CacheDir) GetInvalid(key ...CacheKey) (ts time.Time) {
+ invKey := append(invalidPath, key...)
+
+ stat, _ := cd.Stat(invKey...)
+ return stat.ModTime()
+}
+
+// Checks if the given key is not marked as invalid, or if it is,
+// checks if it was marked more than maxDuration time ago.
+//
+// Calls UnsetInvalid if the keys are valid.
+func (cd *CacheDir) IsValid(maxDuration time.Duration, key ...CacheKey) bool {
+ ts := cd.GetInvalid(key...)
+
+ switch {
+ case ts.IsZero():
+ return true
+ case time.Now().Sub(ts) > maxDuration:
+ cd.UnsetInvalid(key...)
+ return true
+ default:
+ return false
+ }
+}
+
+// Deletes the given key and caches it as invalid.
+func (cd *CacheDir) SetInvalid(key ...CacheKey) error {
+ invKey := append(invalidPath, key...)
+
+ cd.Delete(key...)
+ return cd.Touch(invKey...)
+}
+
+// Removes the given key from the invalid key cache.
+func (cd *CacheDir) UnsetInvalid(key ...CacheKey) error {
+ invKey := append(invalidPath, key...)
+
+ return cd.Delete(invKey...)
+}
--- /dev/null
+package fscache
+
+import (
+ "github.com/Kovensky/go-fscache/lock"
+)
+
+// Locks the file that backs the given key.
+//
+// If the call is successful, it's the caller's responsibility to call Unlock on the returned lock.
+func (cd *CacheDir) Lock(key ...CacheKey) (lock.FileLock, error) {
+ l, err := lock.LockFile(cd.cachePath(key...))
+ if l != nil {
+ l.Lock()
+ }
+ return l, err
+}
--- /dev/null
+package fscache
+
+import (
+ "os"
+ "time"
+)
+
+// Calls os.Stat() on the file or folder that backs the given key.
+func (cd *CacheDir) Stat(key ...CacheKey) (stat os.FileInfo, err error) {
+ lock, err := cd.Lock(key...)
+ if err != nil {
+ return
+ }
+ defer lock.Unlock()
+
+ fh, err := cd.Open(key...)
+ if err != nil {
+ return
+ }
+ defer fh.Close()
+
+ return fh.Stat()
+}
+
+// Updates the mtime of the file backing the given key.
+//
+// Creates an empty file if it doesn't exist.
+func (cd *CacheDir) Touch(key ...CacheKey) (err error) {
+ lock, err := cd.Lock(key...)
+ switch {
+ case err == nil:
+ // AOK
+ defer lock.Unlock()
+ case os.IsNotExist(err):
+ // new file
+ default:
+ return
+ }
+
+ if err = os.Chtimes(cd.cachePath(key...), time.Now(), time.Now()); err == nil {
+ return
+ }
+
+ fh, err := cd.OpenFlags(os.O_APPEND|os.O_CREATE, key...)
+ switch {
+ case err == nil:
+ // AOK
+ case os.IsNotExist(err): // directory path not created yet
+ fh, err = cd.Create(key...)
+ if err != nil {
+ return
+ }
+ default:
+ return
+ }
+ return fh.Close()
+}
--- /dev/null
+package fscache_test
+
+import (
+ "github.com/Kovensky/go-fscache"
+ "math/rand"
+ "testing"
+ "time"
+)
+
+func init() {
+ rand.Seed(time.Now().UnixNano())
+}
+
+func TestCache_Stat(T *testing.T) {
+ cd, err := fscache.NewCacheDir(".testdir")
+ if err != nil {
+ T.Fatal(err)
+ return
+ }
+
+ err = cd.Touch("test", "stat")
+ if err != nil {
+ T.Fatal(err)
+ return
+ }
+
+ stat, err := cd.Stat("test", "stat")
+ if err != nil {
+ T.Fatal(err)
+ return
+ }
+
+ T.Log("IsDir:", stat.IsDir())
+ T.Log("ModTime:", stat.ModTime())
+ T.Log("Name: " + stat.Name())
+ T.Log("Size:", stat.Size(), "bytes")
+
+ if stat.IsDir() || stat.ModTime().IsZero() || stat.Name() != "stat" || stat.Size() != 0 {
+ T.Error("Stat returned unexpected data")
+ }
+}
+
+func TestCache_Touch(T *testing.T) {
+ cd, err := fscache.NewCacheDir(".testdir")
+ if err != nil {
+ T.Fatal(err)
+ return
+ }
+
+ err = cd.Touch("test", "touch", "file")
+ if err != nil {
+ T.Fatal(err)
+ return
+ }
+
+ stat, err := cd.Stat("test", "touch")
+ if err != nil {
+ T.Fatal(err)
+ return
+ }
+
+ if !stat.IsDir() {
+ T.Error("Expected touch to be a dir, file found")
+ }
+
+ // can be anything -- just make a nonblank file
+ size, err := cd.Set(stat.ModTime(), "test", "touch", "subdir", "file")
+ if err != nil {
+ T.Fatal(err)
+ return
+ }
+
+ stat, err = cd.Stat("test", "touch", "subdir", "file")
+ if err != nil {
+ T.Fatal(err)
+ return
+ }
+ if stat.Size() != size {
+ panic("File modified outside of test")
+ }
+
+ T.Log("Waiting 5ms")
+ time.Sleep(5 * time.Millisecond)
+
+ err = cd.Touch("test", "touch", "subdir", "file")
+ if err != nil {
+ T.Fatal(err)
+ }
+ stat2, err := cd.Stat("test", "touch", "subdir", "file")
+
+ if stat.Size() != stat2.Size() {
+ T.Errorf("Touch modified the size")
+ }
+ if !stat2.ModTime().After(stat.ModTime()) {
+ T.Errorf("Touch did not update timestamp (FAT filesystem?)")
+ }
+}
--- /dev/null
+// Cache that uses the filesystem for indexing keys.
+//
+// Prefer running on case-sensitive filesystems; running on a
+// case-insensitive one has the side-effect that keys are also
+// case-insensitive.
+package fscache
--- /dev/null
+// Wrapper for github.com/tgulacsi/go-locking since it doesn't compile on windows.
+//
+// On windows, returns a dummy lock that always succeeds. On other OSes,
+// returns a *locking.FLock.
+//
+// Windows also does file locking on its own, but with different
+// semantics.
+package lock
+
+type FileLock interface {
+ Lock() error
+ Unlock() error
+}
--- /dev/null
+// +build !windows
+
+package lock
+
+import "github.com/tgulacsi/go-locking"
+
+type flockLock struct {
+ locking.FLock
+}
+
+func LockFile(p string) (FileLock, error) {
+ flock, err := locking.NewFLock(p)
+ if err == nil {
+ return &flockLock{FLock: flock}, nil
+ }
+ return nil, err
+}
+
+func (fl *flockLock) Lock() error {
+ return fl.FLock.Lock()
+}
+
+func (fl *flockLock) Unlock() error {
+ return fl.FLock.Unlock()
+}
--- /dev/null
+package lock
+
+func LockFile(p string) (FileLock, error) {
+ return &winLock{}, nil
+}
+
+type winLock struct{}
+
+func (_ *winLock) Lock() error { return nil }
+func (_ *winLock) Unlock() error { return nil }
projects / go-fscache.git / commitdiff