From a5f59e06d99e0f95c10c6e03b8270d35edf56ca6 Mon Sep 17 00:00:00 2001 From: "Diogo Franco (Kovensky)" Date: Tue, 16 Jul 2013 19:47:39 -0300 Subject: [PATCH] Convert cache system to github.com/Kovensky/go-fscache Includes wrapper methods that keep the Cached timestamp more-or-less synchronized with the cache. Internally assert that the objects that are supposed to have embedded timestamps implement the right interface. Changes intent map to use interface{}, as there's no reason for the intents to have to be cacheable. --- animecache.go | 44 ++++---- cache.go | 269 ++++------------------------------------------- cache_test.go | 82 --------------- episodecache.go | 38 ++++--- filecache.go | 80 ++++++-------- fileepcache.go | 52 ++++----- flock.go | 8 -- flock_other.go | 31 ------ flock_windows.go | 12 --- groupcache.go | 97 +++++++---------- intent.go | 40 ++++--- titlecache.go | 22 ++-- udp.go | 33 +++--- 13 files changed, 194 insertions(+), 614 deletions(-) delete mode 100644 cache_test.go delete mode 100644 flock.go delete mode 100644 flock_other.go delete mode 100644 flock_windows.go diff --git a/animecache.go b/animecache.go index 25624fb..4ec6e0e 100644 --- a/animecache.go +++ b/animecache.go @@ -1,11 +1,11 @@ package anidb import ( - "encoding/gob" "fmt" "github.com/Kovensky/go-anidb/http" "github.com/Kovensky/go-anidb/misc" "github.com/Kovensky/go-anidb/udp" + "github.com/Kovensky/go-fscache" "log" "sort" "strconv" @@ -13,13 +13,10 @@ import ( "time" ) -func init() { - gob.RegisterName("*github.com/Kovensky/go-anidb.Anime", &Anime{}) - gob.RegisterName("github.com/Kovensky/go-anidb.AID", AID(0)) -} +var _ cacheable = &Anime{} -func (a *Anime) Touch() { - a.Cached = time.Now() +func (a *Anime) setCachedTS(ts time.Time) { + a.Cached = ts } func (a *Anime) IsStale() bool { @@ -42,15 +39,10 @@ func (a *Anime) IsStale() bool { // Unique Anime IDentifier. type AID int -// make AID Cacheable - -func (e AID) Touch() {} -func (e AID) IsStale() bool { return false } - // Returns a cached Anime. Returns nil if there is no cached Anime with this AID. func (aid AID) Anime() *Anime { var a Anime - if cache.Get(&a, "aid", aid) == nil { + if CacheGet(&a, "aid", aid) == nil { return &a } return nil @@ -64,7 +56,7 @@ type httpAnimeResponse struct { // Retrieves an Anime by its AID. Uses both the HTTP and UDP APIs, // but can work without the UDP API. func (adb *AniDB) AnimeByID(aid AID) <-chan *Anime { - keys := []cacheKey{"aid", aid} + key := []fscache.CacheKey{"aid", aid} ch := make(chan *Anime, 1) if aid < 1 { @@ -72,20 +64,20 @@ func (adb *AniDB) AnimeByID(aid AID) <-chan *Anime { close(ch) } - ic := make(chan Cacheable, 1) + ic := make(chan notification, 1) go func() { ch <- (<-ic).(*Anime); close(ch) }() - if intentMap.Intent(ic, keys...) { + if intentMap.Intent(ic, key...) { return ch } - if !cache.CheckValid(keys...) { - intentMap.NotifyClose((*Anime)(nil), keys...) + if !Cache.IsValid(InvalidKeyCacheDuration, key...) { + intentMap.NotifyClose((*Anime)(nil), key...) return ch } anime := aid.Anime() if !anime.IsStale() { - intentMap.NotifyClose(anime, keys...) + intentMap.NotifyClose(anime, key...) return ch } @@ -136,13 +128,13 @@ func (adb *AniDB) AnimeByID(aid AID) <-chan *Anime { } else { // HTTP ok but parsing not ok if anime.PrimaryTitle == "" { - cache.MarkInvalid(keys...) + Cache.SetInvalid(key...) } switch resp.anime.Error { case "Anime not found", "aid Missing or Invalid": // deleted AID? - cache.Delete(keys...) + Cache.Delete(key...) } ok = false @@ -152,9 +144,9 @@ func (adb *AniDB) AnimeByID(aid AID) <-chan *Anime { httpChan = nil case reply := <-udpChan: if reply.Code() == 330 { - cache.MarkInvalid(keys...) + Cache.SetInvalid(key...) // deleted AID? - cache.Delete(keys...) + Cache.Delete(key...) ok = false break Loop @@ -166,11 +158,11 @@ func (adb *AniDB) AnimeByID(aid AID) <-chan *Anime { } if anime.PrimaryTitle != "" { if ok { - cache.Set(anime, keys...) + CacheSet(anime, key...) } - intentMap.NotifyClose(anime, keys...) + intentMap.NotifyClose(anime, key...) } else { - intentMap.NotifyClose((*Anime)(nil), keys...) + intentMap.NotifyClose((*Anime)(nil), key...) } }() return ch diff --git a/cache.go b/cache.go index bdbe6f8..ed21033 100644 --- a/cache.go +++ b/cache.go @@ -1,280 +1,49 @@ package anidb import ( - "bytes" - "compress/gzip" - "encoding/gob" - "errors" - "fmt" - "io" + "github.com/Kovensky/go-fscache" "os" "path" - "reflect" - "regexp" - "sync" "time" ) -type Cacheable interface { - // Updates the last modified time - Touch() - // Returns true if the Cacheable is nil, or if the last modified time is too old. - IsStale() bool -} - -func init() { - gob.RegisterName("*github.com/Kovensky/go-anidb.invalidKeyCache", &invalidKeyCache{}) -} - -type invalidKeyCache struct{ time.Time } - -func (c *invalidKeyCache) Touch() { - c.Time = time.Now() -} -func (c *invalidKeyCache) IsStale() bool { - return time.Now().Sub(c.Time) > InvalidKeyCacheDuration -} - -type cacheDir struct { - *sync.RWMutex - - CacheDir string -} - func init() { - if err := SetCacheDir(path.Join(os.TempDir(), "anidb", "cache")); err != nil { + c, err := fscache.NewCacheDir(path.Join(os.TempDir(), "anidb", "cache")) + if err != nil { panic(err) } -} - -var cache cacheDir - -// Sets the cache directory to the given path. -// -// go-anidb needs a valid cache directory to function, so, during module -// initialization, it uses os.TempDir() to set a default cache dir. -// go-anidb panics if it's unable to set the default cache dir. -func SetCacheDir(path string) (err error) { - m := cache.RWMutex - if m == nil { - m = &sync.RWMutex{} - cache.RWMutex = m - } - cache.Lock() - - if err = os.MkdirAll(path, 0755|os.ModeDir); err != nil { - cache.Unlock() - return err - } + Cache = *c - cache = cacheDir{ - RWMutex: m, - CacheDir: path, - } - - cache.Unlock() RefreshTitles() - return nil -} - -// Returns the current cache dir. -func GetCacheDir() (path string) { - cache.RLock() - defer cache.RUnlock() - - return cache.CacheDir -} - -type cacheKey interface{} - -// All "bad characters" that can't go in Windows paths. -// It's a superset of the "bad characters" on other OSes, so this works. -var badPath = regexp.MustCompile(`[\\/:\*\?\"<>\|]`) - -func stringify(stuff ...cacheKey) []string { - ret := make([]string, len(stuff)) - for i := range stuff { - s := fmt.Sprint(stuff[i]) - ret[i] = badPath.ReplaceAllLiteralString(s, "_") - } - return ret } -// Each key but the last is treated as a directory. -// The last key is treated as a regular file. -// -// This also means that cache keys that are file-backed -// cannot have subkeys. -func cachePath(keys ...cacheKey) string { - parts := append([]string{GetCacheDir()}, stringify(keys...)...) - p := path.Join(parts...) - return p -} - -// Opens the file that backs the specified keys. -func (c *cacheDir) Open(keys ...cacheKey) (fh *os.File, err error) { - subItem := cachePath(keys...) - return os.Open(subItem) -} - -// Creates a new file to back the specified keys. -func (c *cacheDir) Create(keys ...cacheKey) (fh *os.File, err error) { - subItem := cachePath(keys...) - subDir := path.Dir(subItem) - - if err = os.MkdirAll(subDir, 0755|os.ModeDir); err != nil { - return nil, err - } - return os.Create(subItem) -} +var Cache fscache.CacheDir -// Deletes the file that backs the specified keys. -func (c *cacheDir) Delete(keys ...cacheKey) (err error) { - return os.Remove(cachePath(keys...)) +type cacheable interface { + setCachedTS(time.Time) } -// Deletes the specified key and all subkeys. -func (c *cacheDir) DeleteAll(keys ...cacheKey) (err error) { - return os.RemoveAll(cachePath(keys...)) -} - -func (c *cacheDir) Get(v Cacheable, keys ...cacheKey) (err error) { - - val := reflect.ValueOf(v) - if k := val.Kind(); k == reflect.Ptr || k == reflect.Interface { - val = val.Elem() - } - if !val.CanSet() { - // panic because this is an internal coding mistake - panic("(*cacheDir).Get(): given Cacheable is not setable") - } - - flock := lockFile(cachePath(keys...)) - if flock != nil { - flock.Lock() - } - defer func() { - if flock != nil { - flock.Unlock() - } - }() - - fh, err := c.Open(keys...) - if err != nil { - return err - } - - buf := bytes.Buffer{} - if _, err = io.Copy(&buf, fh); err != nil { - fh.Close() - return err - } - if err = fh.Close(); err != nil { - return err - } - - if flock != nil { - flock.Unlock() - flock = nil - } - - gz, err := gzip.NewReader(&buf) +func CacheSet(v interface{}, key ...fscache.CacheKey) (err error) { + now := time.Now() + _, err = Cache.Set(v, key...) if err != nil { return err } - defer func() { - if e := gz.Close(); err == nil { - err = e - } - }() - - switch f := gz.Header.Comment; f { - case "encoding/gob": - dec := gob.NewDecoder(gz) - err = dec.Decode(v) - default: - return errors.New(fmt.Sprintf("Cached data (format %q) is not in a known format", f)) + switch t := v.(type) { + case cacheable: + t.setCachedTS(now) } - return } -func (c *cacheDir) Set(v Cacheable, keys ...cacheKey) (n int64, err error) { - if v := reflect.ValueOf(v); !v.IsValid() { - panic("reflect.ValueOf() returned invaled value") - } else if k := v.Kind(); k == reflect.Ptr || k == reflect.Interface { - if v.IsNil() { - return // no point in saving nil - } - } - - // First we encode to memory -- we don't want to create/truncate a file and put bad data in it. - buf := bytes.Buffer{} - gz, err := gzip.NewWriterLevel(&buf, gzip.BestCompression) +func CacheGet(v interface{}, key ...fscache.CacheKey) (err error) { + ts, err := Cache.Get(v, key...) if err != nil { - return 0, err - } - gz.Header.Comment = "encoding/gob" - - // it doesn't matter if the caller doesn't see this, - // the important part is that the cache does. - v.Touch() - - enc := gob.NewEncoder(gz) - err = enc.Encode(v) - - if e := gz.Close(); err == nil { - err = e - } - - if err != nil { - return 0, err + return err } - - // We have good data, time to actually put it in the cache - if flock := lockFile(cachePath(keys...)); flock != nil { - flock.Lock() - defer flock.Unlock() + switch t := v.(type) { + case cacheable: + t.setCachedTS(ts) } - - fh, err := c.Create(keys...) - if err != nil { - return 0, err - } - defer func() { - if e := fh.Close(); err == nil { - err = e - } - }() - n, err = io.Copy(fh, &buf) return } - -// Checks if the given keys are not marked as invalid. -// -// If the key was marked as invalid but is no longer considered -// so, deletes the invalid marker. -func (c *cacheDir) CheckValid(keys ...cacheKey) bool { - invKeys := append([]cacheKey{"invalid"}, keys...) - inv := invalidKeyCache{} - - if cache.Get(&inv, invKeys...) == nil { - if inv.IsStale() { - cache.Delete(invKeys...) - } else { - return false - } - } - return true -} - -// Deletes the given keys and marks them as invalid. -// -// They are considered invalid for InvalidKeyCacheDuration. -func (c *cacheDir) MarkInvalid(keys ...cacheKey) error { - invKeys := append([]cacheKey{"invalid"}, keys...) - - cache.Delete(keys...) - _, err := cache.Set(&invalidKeyCache{}, invKeys...) - return err -} diff --git a/cache_test.go b/cache_test.go deleted file mode 100644 index fe37555..0000000 --- a/cache_test.go +++ /dev/null @@ -1,82 +0,0 @@ -package anidb - -import ( - "encoding/gob" - "os" - "path" - "reflect" - "testing" -) - -type stringifyVec struct { - result []string - data []cacheKey -} - -func TestStringify(T *testing.T) { - T.Parallel() - - vec := []stringifyVec{ - stringifyVec{[]string{"a"}, []cacheKey{"a"}}, - } - for i, v := range vec { - str := stringify(v.data...) - if !reflect.DeepEqual(v.result, str) { - T.Errorf("Vector #%d: Expected %v, got %v", i+1, v.result, str) - } - } -} - -type cachePathVec struct { - path string - data []cacheKey -} - -var testDir = path.Join(os.TempDir(), "testing", "anidb") - -func init() { SetCacheDir(testDir) } - -func TestCachePath(T *testing.T) { - T.Parallel() - - vec := []cachePathVec{ - cachePathVec{path.Join(testDir, "a"), []cacheKey{"a"}}, - cachePathVec{path.Join(testDir, "b", "c", "d"), []cacheKey{"b", "c", "d"}}, - } - for i, v := range vec { - str := cachePath(v.data...) - - if v.path != str { - T.Errorf("Vector #%d: Expected %v, got %v", i+1, v.path, str) - } - } -} - -type testString string - -func (_ testString) Touch() {} -func (_ testString) IsStale() bool { return false } - -func init() { - gob.Register(testString("")) -} - -func TestCacheRoundtrip(T *testing.T) { - T.Parallel() - - test := testString("some string") - _, err := cache.Set(test, "test", "string") - if err != nil { - T.Fatalf("Error storing: %v", err) - } - - var t2 testString - err = cache.Get(&t2, "test", "string") - if err != nil { - T.Errorf("Error reading: %v", err) - } - - if test != t2 { - T.Errorf("Expected %q, got %q", test, t2) - } -} diff --git a/episodecache.go b/episodecache.go index 32fb01d..727a2a8 100644 --- a/episodecache.go +++ b/episodecache.go @@ -1,18 +1,16 @@ package anidb import ( - "encoding/gob" + "github.com/Kovensky/go-fscache" "strconv" "strings" "time" ) -func init() { - gob.RegisterName("*github.com/Kovensky/go-anidb.Episode", &Episode{}) -} +var _ cacheable = &Episode{} -func (e *Episode) Touch() { - e.Cached = time.Now() +func (e *Episode) setCachedTS(ts time.Time) { + e.Cached = ts } func (e *Episode) IsStale() bool { @@ -28,15 +26,15 @@ type EID int // Retrieves the Episode corresponding to this EID from the cache. func (eid EID) Episode() *Episode { var e Episode - if cache.Get(&e, "eid", eid) == nil { + if CacheGet(&e, "eid", eid) == nil { return &e } return nil } func cacheEpisode(ep *Episode) { - cache.Set(ep.AID, "aid", "by-eid", ep.EID) - cache.Set(ep, "eid", ep.EID) + CacheSet(ep.AID, "aid", "by-eid", ep.EID) + CacheSet(ep, "eid", ep.EID) } // Retrieves an Episode by its EID. @@ -45,7 +43,7 @@ func cacheEpisode(ep *Episode) { // to an Anime query. Otherwise, uses both the HTTP and UDP // APIs to retrieve it. func (adb *AniDB) EpisodeByID(eid EID) <-chan *Episode { - keys := []cacheKey{"eid", eid} + key := []fscache.CacheKey{"eid", eid} ch := make(chan *Episode, 1) if eid < 1 { @@ -54,20 +52,20 @@ func (adb *AniDB) EpisodeByID(eid EID) <-chan *Episode { return ch } - ic := make(chan Cacheable, 1) + ic := make(chan notification, 1) go func() { ch <- (<-ic).(*Episode); close(ch) }() - if intentMap.Intent(ic, keys...) { + if intentMap.Intent(ic, key...) { return ch } - if !cache.CheckValid(keys...) { - intentMap.NotifyClose((*Episode)(nil), keys...) + if !Cache.IsValid(InvalidKeyCacheDuration, key...) { + intentMap.NotifyClose((*Episode)(nil), key...) return ch } e := eid.Episode() if !e.IsStale() { - intentMap.NotifyClose(e, keys...) + intentMap.NotifyClose(e, key...) return ch } @@ -78,7 +76,8 @@ func (adb *AniDB) EpisodeByID(eid EID) <-chan *Episode { // API episode list. aid := AID(0) - ok := cache.Get(&aid, "aid", "by-eid", eid) == nil + _, err := Cache.Get(&aid, "aid", "by-eid", eid) + ok := err == nil udpDone := false @@ -102,8 +101,7 @@ func (adb *AniDB) EpisodeByID(eid EID) <-chan *Episode { break } } else if reply.Code() == 340 { - cache.MarkInvalid(keys...) - cache.Delete(keys...) // deleted EID? + Cache.SetInvalid(key...) break } else { break @@ -119,10 +117,10 @@ func (adb *AniDB) EpisodeByID(eid EID) <-chan *Episode { } else { // the EID<->AID map broke ok = false - cache.Delete("aid", "by-eid", eid) + Cache.Delete("aid", "by-eid", eid) } } - intentMap.NotifyClose(e, keys...) + intentMap.NotifyClose(e, key...) }() return ch } diff --git a/filecache.go b/filecache.go index 3304f48..4d65b16 100644 --- a/filecache.go +++ b/filecache.go @@ -1,10 +1,10 @@ package anidb import ( - "encoding/gob" "fmt" "github.com/Kovensky/go-anidb/misc" "github.com/Kovensky/go-anidb/udp" + "github.com/Kovensky/go-fscache" "image" "log" "regexp" @@ -14,14 +14,10 @@ import ( "time" ) -func init() { - gob.RegisterName("*github.com/Kovensky/go-anidb.File", &File{}) - gob.RegisterName("*github.com/Kovensky/go-anidb.fidCache", &fidCache{}) - gob.RegisterName("github.com/Kovensky/go-anidb.FID", FID(0)) -} +var _ cacheable = &File{} -func (f *File) Touch() { - f.Cached = time.Now() +func (f *File) setCachedTS(ts time.Time) { + f.Cached = ts } func (f *File) IsStale() bool { @@ -34,34 +30,21 @@ func (f *File) IsStale() bool { return time.Now().Sub(f.Cached) > FileCacheDuration } -type FID int - -// make FID Cacheable +func cacheFile(f *File) { + CacheSet(f.FID, "fid", "by-ed2k", f.Ed2kHash, f.Filesize) + CacheSet(f, "fid", f.FID) +} -func (e FID) Touch() {} -func (e FID) IsStale() bool { return false } +type FID int func (fid FID) File() *File { var f File - if cache.Get(&f, "fid", fid) == nil { + if CacheGet(&f, "fid", fid) == nil { return &f } return nil } -type fidCache struct { - FID - Time time.Time -} - -func (c *fidCache) Touch() { - c.Time = time.Now() -} - -func (c *fidCache) IsStale() bool { - return time.Now().Sub(c.Time) > FileCacheDuration -} - // Prefetches the Anime, Episode and Group that this // file is linked to using the given AniDB instance. // @@ -85,7 +68,7 @@ func (f *File) Prefetch(adb *AniDB) <-chan *File { // Retrieves a File by its FID. Uses the UDP API. func (adb *AniDB) FileByID(fid FID) <-chan *File { - keys := []cacheKey{"fid", fid} + key := []fscache.CacheKey{"fid", fid} ch := make(chan *File, 1) @@ -95,20 +78,20 @@ func (adb *AniDB) FileByID(fid FID) <-chan *File { return ch } - ic := make(chan Cacheable, 1) + ic := make(chan notification, 1) go func() { ch <- (<-ic).(*File); close(ch) }() - if intentMap.Intent(ic, keys...) { + if intentMap.Intent(ic, key...) { return ch } - if !cache.CheckValid(keys...) { - intentMap.NotifyClose((*File)(nil), keys...) + if !Cache.IsValid(InvalidKeyCacheDuration, key...) { + intentMap.NotifyClose((*File)(nil), key...) return ch } f := fid.File() if !f.IsStale() { - intentMap.NotifyClose(f, keys...) + intentMap.NotifyClose(f, key...) return ch } @@ -123,13 +106,12 @@ func (adb *AniDB) FileByID(fid FID) <-chan *File { if reply.Error() == nil { f = adb.parseFileResponse(reply, false) - cache.Set(&fidCache{FID: f.FID}, "fid", "by-ed2k", f.Ed2kHash, f.Filesize) - cache.Set(f, keys...) + cacheFile(f) } else if reply.Code() == 320 { - cache.MarkInvalid(keys...) + Cache.SetInvalid(key...) } - intentMap.NotifyClose(f, keys...) + intentMap.NotifyClose(f, key...) }() return ch } @@ -138,7 +120,7 @@ var validEd2kHash = regexp.MustCompile(`\A[:xdigit:]{32}\z`) // Retrieves a File by its Ed2kHash + Filesize combination. Uses the UDP API. func (adb *AniDB) FileByEd2kSize(ed2k string, size int64) <-chan *File { - keys := []cacheKey{"fid", "by-ed2k", ed2k, size} + key := []fscache.CacheKey{"fid", "by-ed2k", ed2k, size} ch := make(chan *File, 1) @@ -150,7 +132,7 @@ func (adb *AniDB) FileByEd2kSize(ed2k string, size int64) <-chan *File { // AniDB always uses lower case hashes ed2k = strings.ToLower(ed2k) - ic := make(chan Cacheable, 1) + ic := make(chan notification, 1) go func() { fid := (<-ic).(FID) if fid > 0 { @@ -158,23 +140,22 @@ func (adb *AniDB) FileByEd2kSize(ed2k string, size int64) <-chan *File { } close(ch) }() - if intentMap.Intent(ic, keys...) { + if intentMap.Intent(ic, key...) { return ch } - if !cache.CheckValid(keys...) { - intentMap.NotifyClose(FID(0), keys...) + if !Cache.IsValid(InvalidKeyCacheDuration, key...) { + intentMap.NotifyClose(FID(0), key...) return ch } fid := FID(0) - var ec fidCache - if cache.Get(&ec, keys...) == nil && !ec.IsStale() { - intentMap.NotifyClose(ec.FID, keys...) + switch ts, err := Cache.Get(&fid, key...); { + case err != nil && time.Now().Sub(ts) < FileCacheDuration: + intentMap.NotifyClose(fid, key...) return ch } - fid = ec.FID go func() { reply := <-adb.udp.SendRecv("FILE", @@ -191,15 +172,14 @@ func (adb *AniDB) FileByEd2kSize(ed2k string, size int64) <-chan *File { fid = f.FID - cache.Set(&fidCache{FID: fid}, keys...) - cache.Set(f, "fid", fid) + cacheFile(f) } else if reply.Code() == 320 { // file not found - cache.MarkInvalid(keys...) + Cache.SetInvalid(key...) } else if reply.Code() == 322 { // multiple files found panic("Don't know what to do with " + strings.Join(reply.Lines(), "\n")) } - intentMap.NotifyClose(fid, keys...) + intentMap.NotifyClose(fid, key...) }() return ch } diff --git a/fileepcache.go b/fileepcache.go index e01db22..6e4b914 100644 --- a/fileepcache.go +++ b/fileepcache.go @@ -1,19 +1,12 @@ package anidb import ( + "github.com/Kovensky/go-fscache" "strconv" "strings" "time" ) -type fidList struct { - FIDs []FID - Time time.Time -} - -func (l *fidList) Touch() { l.Time = time.Now() } -func (l *fidList) IsStale() bool { return time.Now().Sub(l.Time) > FileCacheDuration } - // Gets the Files that the given Group has released for the given // Episode. Convenience method that calls FilesByGID. func (adb *AniDB) FilesByGroup(ep *Episode, g *Group) <-chan *File { @@ -59,7 +52,7 @@ func (adb *AniDB) FilesByGID(ep *Episode, gid GID) <-chan *File { // On API error (offline, etc), the first *File returned is nil, // followed by cached files (which may also be nil). func (adb *AniDB) FIDsByGID(ep *Episode, gid GID) <-chan FID { - keys := []cacheKey{"fid", "by-ep-gid", ep.EID, gid} + key := []fscache.CacheKey{"fid", "by-eid-gid", ep.EID, gid} ch := make(chan FID, 10) @@ -69,30 +62,31 @@ func (adb *AniDB) FIDsByGID(ep *Episode, gid GID) <-chan FID { return ch } - ic := make(chan Cacheable, 1) + ic := make(chan notification, 1) go func() { for c := range ic { ch <- c.(FID) } close(ch) }() - if intentMap.Intent(ic, keys...) { + if intentMap.Intent(ic, key...) { return ch } - if !cache.CheckValid(keys...) { - intentMap.Close(keys...) + if !Cache.IsValid(InvalidKeyCacheDuration, key...) { + intentMap.Close(key...) return ch } - var fids fidList - if cache.Get(&fids, keys...) == nil { - is := intentMap.LockIntent(keys...) + var fids []FID + switch ts, err := Cache.Get(&fids, key...); { + case err == nil && time.Now().Sub(ts) < FileCacheDuration: + is := intentMap.LockIntent(key...) go func() { - defer intentMap.Free(is, keys...) + defer intentMap.Free(is, key...) defer is.Close() - for _, fid := range fids.FIDs { + for _, fid := range fids { is.Notify(fid) } }() @@ -109,33 +103,31 @@ func (adb *AniDB) FIDsByGID(ep *Episode, gid GID) <-chan FID { "amask": fileAmask, }) - is := intentMap.LockIntent(keys...) - defer intentMap.Free(is, keys...) + is := intentMap.LockIntent(key...) + defer intentMap.Free(is, key...) switch reply.Code() { case 220: f := adb.parseFileResponse(reply, true) - fids.FIDs = []FID{f.FID} - cache.Set(&fids, keys...) + fids = []FID{f.FID} + CacheSet(&fids, key...) - cache.Set(&fidCache{FID: f.FID}, "fid", "by-ed2k", f.Ed2kHash, f.Filesize) - cache.Set(f, "fid", f.FID) + cacheFile(f) is.NotifyClose(f.FID) return case 322: parts := strings.Split(reply.Lines()[1], "|") - fids.FIDs = make([]FID, len(parts)) + fids = make([]FID, len(parts)) for i := range parts { id, _ := strconv.ParseInt(parts[i], 10, 32) - fids.FIDs[i] = FID(id) + fids[i] = FID(id) } - cache.Set(&fids, keys...) + CacheSet(&fids, key...) case 320: - cache.MarkInvalid(keys...) - cache.Delete(keys...) + Cache.SetInvalid(key...) is.Close() return default: @@ -143,7 +135,7 @@ func (adb *AniDB) FIDsByGID(ep *Episode, gid GID) <-chan FID { } defer is.Close() - for _, fid := range fids.FIDs { + for _, fid := range fids { is.Notify(fid) } }() diff --git a/flock.go b/flock.go deleted file mode 100644 index 6df0b2b..0000000 --- a/flock.go +++ /dev/null @@ -1,8 +0,0 @@ -package anidb - -type fileLock interface { - Lock() error - Unlock() error -} - -// func lockFile(p path) fileLock diff --git a/flock_other.go b/flock_other.go deleted file mode 100644 index bf275a2..0000000 --- a/flock_other.go +++ /dev/null @@ -1,31 +0,0 @@ -// +build !windows - -package anidb - -import "github.com/tgulacsi/go-locking" - -type flockLock struct { - locking.FLock -} - -func lockFile(p string) fileLock { - flock, err := locking.NewFLock(p) - if err == nil { - return &flockLock{FLock: flock} - } - return nil -} - -func (fl *flockLock) Lock() error { - if fl != nil { - return fl.FLock.Lock() - } - return nil -} - -func (fl *flockLock) Unlock() error { - if fl != nil { - return fl.FLock.Unlock() - } - return nil -} diff --git a/flock_windows.go b/flock_windows.go deleted file mode 100644 index cce4706..0000000 --- a/flock_windows.go +++ /dev/null @@ -1,12 +0,0 @@ -package anidb - -type winFileLock struct{} - -func lockFile(p string) fileLock { - return &winFileLock{} -} - -// empty implementations -- go-locking doesn't support windows -// windows also does file locking on its own -func (_ *winFileLock) Lock() error { return nil } -func (_ *winFileLock) Unlock() error { return nil } diff --git a/groupcache.go b/groupcache.go index 8d0f92c..cbf1c1a 100644 --- a/groupcache.go +++ b/groupcache.go @@ -1,22 +1,18 @@ package anidb import ( - "encoding/gob" "github.com/Kovensky/go-anidb/http" "github.com/Kovensky/go-anidb/udp" + "github.com/Kovensky/go-fscache" "strconv" "strings" "time" ) -func init() { - gob.RegisterName("*github.com/Kovensky/go-anidb.Group", &Group{}) - gob.RegisterName("github.com/Kovensky/go-anidb.GID", GID(0)) - gob.RegisterName("*github.com/Kovensky/go-anidb.gidCache", &gidCache{}) -} +var _ cacheable = &Group{} -func (g *Group) Touch() { - g.Cached = time.Now() +func (g *Group) setCachedTS(ts time.Time) { + g.Cached = ts } func (g *Group) IsStale() bool { @@ -29,36 +25,24 @@ func (g *Group) IsStale() bool { // Unique Group IDentifier type GID int -// make GID cacheable - -func (e GID) Touch() {} -func (e GID) IsStale() bool { return false } +func cacheGroup(g *Group) { + CacheSet(g.GID, "gid", "by-name", g.Name) + CacheSet(g.GID, "gid", "by-shortname", g.ShortName) + CacheSet(g, "gid", g.GID) +} // Retrieves the Group from the cache. func (gid GID) Group() *Group { var g Group - if cache.Get(&g, "gid", gid) == nil { + if CacheGet(&g, "gid", gid) == nil { return &g } return nil } -type gidCache struct { - GID - Time time.Time -} - -func (c *gidCache) Touch() { c.Time = time.Now() } -func (c *gidCache) IsStale() bool { - if c != nil && time.Now().Sub(c.Time) < GroupCacheDuration { - return false - } - return true -} - // Retrieves a Group by its GID. Uses the UDP API. func (adb *AniDB) GroupByID(gid GID) <-chan *Group { - keys := []cacheKey{"gid", gid} + key := []fscache.CacheKey{"gid", gid} ch := make(chan *Group, 1) if gid < 1 { @@ -67,20 +51,20 @@ func (adb *AniDB) GroupByID(gid GID) <-chan *Group { return ch } - ic := make(chan Cacheable, 1) + ic := make(chan notification, 1) go func() { ch <- (<-ic).(*Group); close(ch) }() - if intentMap.Intent(ic, keys...) { + if intentMap.Intent(ic, key...) { return ch } - if !cache.CheckValid(keys...) { - intentMap.NotifyClose((*Group)(nil), keys...) + if !Cache.IsValid(InvalidKeyCacheDuration, key...) { + intentMap.NotifyClose((*Group)(nil), key...) return ch } g := gid.Group() if !g.IsStale() { - intentMap.NotifyClose(g, keys...) + intentMap.NotifyClose(g, key...) return ch } @@ -91,15 +75,12 @@ func (adb *AniDB) GroupByID(gid GID) <-chan *Group { if reply.Error() == nil { g = parseGroupReply(reply) - cache.Set(&gidCache{GID: g.GID}, "gid", "by-name", g.Name) - cache.Set(&gidCache{GID: g.GID}, "gid", "by-shortname", g.ShortName) - cache.Set(g, keys...) + cacheGroup(g) } else if reply.Code() == 350 { - cache.MarkInvalid(keys...) - cache.Delete(keys...) // deleted group? + Cache.SetInvalid(key...) } - intentMap.NotifyClose(g, keys...) + intentMap.NotifyClose(g, key...) }() return ch } @@ -107,8 +88,8 @@ func (adb *AniDB) GroupByID(gid GID) <-chan *Group { // Retrieves a Group by its name. Either full or short names are matched. // Uses the UDP API. func (adb *AniDB) GroupByName(gname string) <-chan *Group { - keys := []cacheKey{"gid", "by-name", gname} - altKeys := []cacheKey{"gid", "by-shortname", gname} + key := []fscache.CacheKey{"gid", "by-name", gname} + altKey := []fscache.CacheKey{"gid", "by-shortname", gname} ch := make(chan *Group, 1) if gname == "" { @@ -117,7 +98,7 @@ func (adb *AniDB) GroupByName(gname string) <-chan *Group { return ch } - ic := make(chan Cacheable, 1) + ic := make(chan notification, 1) go func() { gid := (<-ic).(GID) if gid > 0 { @@ -125,30 +106,27 @@ func (adb *AniDB) GroupByName(gname string) <-chan *Group { } close(ch) }() - if intentMap.Intent(ic, keys...) { + if intentMap.Intent(ic, key...) { return ch } - if !cache.CheckValid(keys...) { - intentMap.NotifyClose(GID(0), keys...) + if !Cache.IsValid(InvalidKeyCacheDuration, key...) { + intentMap.NotifyClose(GID(0), key...) return ch } gid := GID(0) - var gc gidCache - if cache.Get(&gc, keys...) == nil && !gc.IsStale() { - intentMap.NotifyClose(gc.GID, keys...) + switch ts, err := Cache.Get(&gid, key...); { + case err == nil && time.Now().Sub(ts) < GroupCacheDuration: + intentMap.NotifyClose(gid, key...) return ch - } - gid = gc.GID - - if gid == 0 { - if cache.Get(&gc, altKeys...) == nil && !gc.IsStale() { - intentMap.NotifyClose(gc.GID, keys...) + default: + switch ts, err = Cache.Get(&gid, altKey...); { + case err == nil && time.Now().Sub(ts) < GroupCacheDuration: + intentMap.NotifyClose(gid, key...) return ch } - gid = gc.GID } go func() { @@ -161,16 +139,13 @@ func (adb *AniDB) GroupByName(gname string) <-chan *Group { gid = g.GID - cache.Set(&gidCache{GID: gid}, keys...) - cache.Set(&gidCache{GID: gid}, altKeys...) - cache.Set(g, "gid", gid) + cacheGroup(g) } else if reply.Code() == 350 { - cache.MarkInvalid(keys...) - cache.Delete(keys...) // renamed group? - cache.Delete(altKeys...) + Cache.SetInvalid(key...) + Cache.SetInvalid(altKey...) } - intentMap.NotifyClose(gid, keys...) + intentMap.NotifyClose(gid, key...) }() return ch } diff --git a/intent.go b/intent.go index 56900a4..88b5a3f 100644 --- a/intent.go +++ b/intent.go @@ -1,10 +1,16 @@ package anidb -import "sync" +import ( + "github.com/Kovensky/go-fscache" + "strings" + "sync" +) + +type notification interface{} type intentStruct struct { sync.Mutex - chs []chan Cacheable + chs []chan notification } type intentMapStruct struct { @@ -18,6 +24,10 @@ var intentMap = &intentMapStruct{ m: map[string]*intentStruct{}, } +func intentKey(key ...fscache.CacheKey) string { + return strings.Join(fscache.Stringify(key...), "-") +} + // Register a channel to be notified when the specified keys are notified. // Returns whether the caller was the first to register intent for the given // keys. @@ -25,8 +35,8 @@ var intentMap = &intentMapStruct{ // Cache checks should be done after registering intent, since it's possible to // register Intent while a Notify is running, and the Notify is done after // setting the cache. -func (m *intentMapStruct) Intent(ch chan Cacheable, keys ...cacheKey) bool { - key := cachePath(keys...) +func (m *intentMapStruct) Intent(ch chan notification, keys ...fscache.CacheKey) bool { + key := intentKey(keys...) m.intentLock.Lock() defer m.intentLock.Unlock() @@ -55,15 +65,15 @@ func (m *intentMapStruct) Intent(ch chan Cacheable, keys ...cacheKey) bool { // // The intentStruct can be directly unlocked, or given to Free to also // remove it from the intent map. -func (m *intentMapStruct) LockIntent(keys ...cacheKey) *intentStruct { +func (m *intentMapStruct) LockIntent(keys ...fscache.CacheKey) *intentStruct { m.Lock() defer m.Unlock() return m._lockIntent(keys...) } -func (m *intentMapStruct) _lockIntent(keys ...cacheKey) *intentStruct { - s, ok := m.m[cachePath(keys...)] +func (m *intentMapStruct) _lockIntent(keys ...fscache.CacheKey) *intentStruct { + s, ok := m.m[intentKey(keys...)] if !ok { return nil } @@ -73,16 +83,16 @@ func (m *intentMapStruct) _lockIntent(keys ...cacheKey) *intentStruct { } // Removes the given intent from the intent map and unlocks the intentStruct. -func (m *intentMapStruct) Free(is *intentStruct, keys ...cacheKey) { +func (m *intentMapStruct) Free(is *intentStruct, keys ...fscache.CacheKey) { m.Lock() defer m.Unlock() m._free(is, keys...) } -func (m *intentMapStruct) _free(is *intentStruct, keys ...cacheKey) { +func (m *intentMapStruct) _free(is *intentStruct, keys ...fscache.CacheKey) { // deletes the key before unlocking, Intent needs to recheck key status - delete(m.m, cachePath(keys...)) + delete(m.m, intentKey(keys...)) // better than unlocking then deleting -- could delete a "brand new" entry is.Unlock() } @@ -91,7 +101,7 @@ func (m *intentMapStruct) _free(is *intentStruct, keys ...cacheKey) { // also removes them from the intent map. // // Should be called after setting the cache. -func (m *intentMapStruct) NotifyClose(v Cacheable, keys ...cacheKey) { +func (m *intentMapStruct) NotifyClose(v notification, keys ...fscache.CacheKey) { m.Lock() defer m.Unlock() @@ -103,7 +113,7 @@ func (m *intentMapStruct) NotifyClose(v Cacheable, keys ...cacheKey) { // Closes all channels that are listening for the specified keys // and removes them from the intent map. -func (m *intentMapStruct) Close(keys ...cacheKey) { +func (m *intentMapStruct) Close(keys ...fscache.CacheKey) { m.Lock() defer m.Unlock() @@ -115,7 +125,7 @@ func (m *intentMapStruct) Close(keys ...cacheKey) { // Notifies all channels that are listening for the specified keys, // but doesn't close or remove them from the intent map. -func (m *intentMapStruct) Notify(v Cacheable, keys ...cacheKey) { +func (m *intentMapStruct) Notify(v notification, keys ...fscache.CacheKey) { m.Lock() defer m.Unlock() @@ -126,7 +136,7 @@ func (m *intentMapStruct) Notify(v Cacheable, keys ...cacheKey) { } // NOTE: does not lock the stuct -func (s *intentStruct) Notify(v Cacheable) { +func (s *intentStruct) Notify(v notification) { for _, ch := range s.chs { ch <- v } @@ -141,7 +151,7 @@ func (s *intentStruct) Close() { } // NOTE: does not lock the struct -func (s *intentStruct) NotifyClose(v Cacheable) { +func (s *intentStruct) NotifyClose(v notification) { for _, ch := range s.chs { ch <- v close(ch) diff --git a/titlecache.go b/titlecache.go index 5901fd4..a69a5e4 100644 --- a/titlecache.go +++ b/titlecache.go @@ -6,6 +6,7 @@ import ( "io" "log" "net/http" + "os" "time" ) @@ -13,12 +14,13 @@ var titlesDB = &titles.TitlesDatabase{} // Loads the database from anime-titles.dat.gz in the cache dir. func RefreshTitles() error { - if flock := lockFile(cachePath("anime-titles.dat.gz")); flock != nil { - flock.Lock() - defer flock.Unlock() + if lock, err := Cache.Lock("anime-titles.dat.gz"); err != nil { + return err + } else { + defer lock.Unlock() } - fh, err := cache.Open("anime-titles.dat.gz") + fh, err := Cache.Open("anime-titles.dat.gz") if err != nil { return err } @@ -42,9 +44,13 @@ func UpdateTitles() error { return nil } - if flock := lockFile(cachePath("anime-titles.dat.gz")); flock != nil { - flock.Lock() - defer flock.Unlock() + switch lock, err := Cache.Lock("anime-titles.dat.gz"); { + case os.IsNotExist(err): + // we're creating it now + case err == nil: + defer lock.Unlock() + default: + return err } c := &http.Client{Transport: &http.Transport{DisableCompression: true}} @@ -67,7 +73,7 @@ func UpdateTitles() error { return err } - fh, err := cache.Create("anime-titles.dat.gz") + fh, err := Cache.Create("anime-titles.dat.gz") if err != nil { return err } diff --git a/udp.go b/udp.go index 89b39d8..59d6c27 100644 --- a/udp.go +++ b/udp.go @@ -1,42 +1,33 @@ package anidb import ( - "encoding/gob" "github.com/Kovensky/go-anidb/udp" "log" "sync" "time" ) -func init() { - gob.RegisterName("*github.com/Kovensky/go-anidb.banCache", &banCache{}) -} - const banDuration = 30*time.Minute + 1*time.Second -type banCache struct{ time.Time } - -func (c *banCache) Touch() { - c.Time = time.Now() -} -func (c *banCache) IsStale() bool { - return time.Now().Sub(c.Time) > banDuration -} - // Returns whether the last UDP API access returned a 555 BANNED message. func Banned() bool { - var banTime banCache - cache.Get(&banTime, "banned") + stat, err := Cache.Stat("banned") + if err != nil { + return false + } - stale := banTime.IsStale() - if stale { - cache.Delete("banned") + switch ts := stat.ModTime(); { + case ts.IsZero(): + return false + case time.Now().Sub(ts) > banDuration: + return false + default: + return true } - return !stale } func setBanned() { - cache.Set(&banCache{}, "banned") + Cache.Touch("banned") } type paramSet struct { -- 2.44.0