X-Git-Url: https://git.lizzy.rs/?a=blobdiff_plain;f=filecache.go;h=ae529e3594362c5575ebfb19ebda3d789b35a267;hb=57d0119266f3ca005a0cfc12ca96ca2a8c10c93b;hp=e3ae93c03911d5f4d84fdcf5373fc89e1a0c0476;hpb=36351396fa6db3dfc295b63fd1f3f267b0d82c30;p=go-anidb.git diff --git a/filecache.go b/filecache.go index e3ae93c..ae529e3 100644 --- a/filecache.go +++ b/filecache.go @@ -1,24 +1,22 @@ package anidb import ( - "encoding/gob" "fmt" "github.com/Kovensky/go-anidb/misc" "github.com/Kovensky/go-anidb/udp" + "github.com/Kovensky/go-fscache" "image" - "log" + "regexp" + "sort" "strconv" "strings" - "sync" "time" ) -func init() { - gob.RegisterName("*github.com/Kovensky/go-anidb.File", &File{}) -} +var _ cacheable = &File{} -func (f *File) Touch() { - f.Cached = time.Now() +func (f *File) setCachedTS(ts time.Time) { + f.Cached = ts } func (f *File) IsStale() bool { @@ -31,23 +29,20 @@ func (f *File) IsStale() bool { return time.Now().Sub(f.Cached) > FileCacheDuration } -type FID int - -func (fid FID) File() *File { - f, _ := caches.Get(fileCache).Get(int(fid)).(*File) - return f +func cacheFile(f *File) { + CacheSet(f.AID, "aid", "by-eid", f.EID) + CacheSet(f.FID, "fid", "by-ed2k", f.Ed2kHash, f.Filesize) + CacheSet(f, "fid", f.FID) } -func ed2kKey(ed2k string, size int64) string { - return fmt.Sprintf("%s-%016x", ed2k, size) -} +type FID int -func ed2kCache(f *File) { - if f != nil { - ed2kFidLock.Lock() - defer ed2kFidLock.Unlock() - ed2kFidMap[ed2kKey(f.Ed2kHash, f.Filesize)] = f.FID +func (fid FID) File() *File { + var f File + if CacheGet(&f, "fid", fid) == nil { + return &f } + return nil } // Prefetches the Anime, Episode and Group that this @@ -71,22 +66,32 @@ func (f *File) Prefetch(adb *AniDB) <-chan *File { return ch } -var ed2kFidMap = map[string]FID{} -var ed2kIntent = map[string][]chan *File{} -var ed2kFidLock = sync.RWMutex{} - +// Retrieves a File by its FID. Uses the UDP API. func (adb *AniDB) FileByID(fid FID) <-chan *File { + key := []fscache.CacheKey{"fid", fid} + ch := make(chan *File, 1) - if f := fid.File(); !f.IsStale() { - ch <- f + + if fid < 1 { + ch <- nil close(ch) return ch } - fc := caches.Get(fileCache) - ic := make(chan Cacheable, 1) + ic := make(chan notification, 1) go func() { ch <- (<-ic).(*File); close(ch) }() - if fc.Intent(int(fid), ic) { + if intentMap.Intent(ic, key...) { + return ch + } + + if !Cache.IsValid(InvalidKeyCacheDuration, key...) { + intentMap.NotifyClose((*File)(nil), key...) + return ch + } + + f := fid.File() + if !f.IsStale() { + intentMap.NotifyClose(f, key...) return ch } @@ -98,38 +103,58 @@ func (adb *AniDB) FileByID(fid FID) <-chan *File { "amask": fileAmask, }) - var f *File if reply.Error() == nil { - f = parseFileResponse(reply) + adb.parseFileResponse(&f, reply, false) + + cacheFile(f) + } else if reply.Code() == 320 { + Cache.SetInvalid(key...) } - ed2kCache(f) - fc.Set(int(fid), f) + + intentMap.NotifyClose(f, key...) }() return ch } +var validEd2kHash = regexp.MustCompile(`\A[[:xdigit:]]{32}\z`) + +// Retrieves a File by its Ed2kHash + Filesize combination. Uses the UDP API. func (adb *AniDB) FileByEd2kSize(ed2k string, size int64) <-chan *File { - key := ed2kKey(ed2k, size) + key := []fscache.CacheKey{"fid", "by-ed2k", ed2k, size} + ch := make(chan *File, 1) - ed2kFidLock.RLock() - if fid, ok := ed2kFidMap[key]; ok { - ed2kFidLock.RUnlock() - if f := fid.File(); f != nil { - ch <- f - close(ch) - return ch + if size < 1 || !validEd2kHash.MatchString(ed2k) { + ch <- nil + close(ch) + return ch + } + // AniDB always uses lower case hashes + ed2k = strings.ToLower(ed2k) + + ic := make(chan notification, 1) + go func() { + fid := (<-ic).(FID) + if fid > 0 { + ch <- <-adb.FileByID(fid) } - return adb.FileByID(fid) + close(ch) + }() + if intentMap.Intent(ic, key...) { + return ch } - ed2kFidLock.RUnlock() - ed2kFidLock.Lock() - if list, ok := ed2kIntent[key]; ok { - ed2kIntent[key] = append(list, ch) + if !Cache.IsValid(InvalidKeyCacheDuration, key...) { + intentMap.NotifyClose(FID(0), key...) + return ch + } + + fid := FID(0) + + switch ts, err := Cache.Get(&fid, key...); { + case err == nil && time.Now().Sub(ts) < FileCacheDuration: + intentMap.NotifyClose(fid, key...) return ch - } else { - ed2kIntent[key] = append(list, ch) } go func() { @@ -143,42 +168,34 @@ func (adb *AniDB) FileByEd2kSize(ed2k string, size int64) <-chan *File { var f *File if reply.Error() == nil { - f = parseFileResponse(reply) + adb.parseFileResponse(&f, reply, false) + + fid = f.FID - ed2kCache(f) - caches.Get(fileCache).Set(int(f.FID), f) + cacheFile(f) } else if reply.Code() == 320 { // file not found - ed2kFidLock.Lock() - delete(ed2kFidMap, key) - ed2kFidLock.Unlock() + Cache.SetInvalid(key...) } else if reply.Code() == 322 { // multiple files found panic("Don't know what to do with " + strings.Join(reply.Lines(), "\n")) } - ed2kFidLock.Lock() - defer ed2kFidLock.Unlock() - - for _, ch := range ed2kIntent[key] { - ch <- f - close(ch) - } - delete(ed2kIntent, key) + intentMap.NotifyClose(fid, key...) }() return ch } -var fileFmask = "77da7fe8" +var fileFmask = "7fda7fe8" var fileAmask = "00008000" const ( - fileStateCRCOK = 1 << iota - fileStateCRCERR - fileStateV2 - fileStateV3 - fileStateV4 - fileStateV5 - fileStateUncensored - fileStateCensored + stateCRCOK = 1 << iota + stateCRCERR + stateV2 + stateV3 + stateV4 + stateV5 + stateUncensored + stateCensored ) func sanitizeCodec(codec string) string { @@ -195,40 +212,179 @@ func sanitizeCodec(codec string) string { return codec } -func parseFileResponse(reply udpapi.APIReply) *File { +var opedRE = regexp.MustCompile(`\A(Opening|Ending)(?: (\d+))?\z`) + +func (adb *AniDB) parseFileResponse(f **File, reply udpapi.APIReply, calledFromFIDsByGID bool) bool { if reply.Error() != nil { - return nil + return false } if reply.Truncated() { panic("Truncated") } + uidChan := make(chan UID, 1) + if adb.udp.credentials != nil { + go func() { uidChan <- <-adb.GetUserUID(decrypt(adb.udp.credentials.username)) }() + } else { + uidChan <- 0 + close(uidChan) + } + parts := strings.Split(reply.Lines()[1], "|") ints := make([]int64, len(parts)) for i, p := range parts { - ints[i], _ = strconv.ParseInt(parts[i], 10, 64) - log.Printf("#%d: %s\n", i, p) + ints[i], _ = strconv.ParseInt(p, 10, 64) } - // how does epno look like? - log.Println("epno: " + parts[23]) + partial := false + + rels := strings.Split(parts[5], "'") + relList := make([]EID, 0, len(parts[5])) + related := make(RelatedEpisodes, len(parts[5])) + for _, rel := range rels { + r := strings.Split(rel, ",") + if len(r) < 2 { + continue + } + + eid, _ := strconv.ParseInt(r[0], 10, 32) + pct, _ := strconv.ParseInt(r[1], 10, 32) + relList = append(relList, EID(eid)) + related[EID(eid)] = float32(pct) / 100 + + if pct != 100 { + partial = true + } + } + + epno := misc.ParseEpisodeList(parts[24]) + fid := FID(ints[0]) + aid := AID(ints[1]) + eid := EID(ints[2]) + gid := GID(ints[3]) + lid := LID(ints[4]) + + if !epno[0].Start.ContainsEpisodes(epno[0].End) || len(epno) > 1 || len(relList) > 0 { + // epno is broken -- we need to sanitize it + thisEp := <-adb.EpisodeByID(eid) + bad := false + if thisEp != nil { + parts := make([]string, 1, len(relList)+1) + parts[0] = thisEp.Episode.String() + + // everything after this SHOULD be cache hits now, unless this is somehow + // linked with an EID from a different anime (*stares at Haruhi*). + // We don't want to use eps from different AIDs anyway, so that makes + // the job easier. + + // We check if the related episodes are all in sequence from this one. + // If they are, we build a new epno with the sequence. Otherwise, + // our epno will only have the primary episode. + + // gather the episode numbers + for _, eid := range relList { + if ep := eid.Episode(); ep != nil && ep.AID == thisEp.AID { + parts = append(parts, ep.Episode.String()) + } else { + bad = true + break + } + } + + test := misc.EpisodeList{} + // only if we didn't break the loop + if !bad { + test = misc.ParseEpisodeList(strings.Join(parts, ",")) + } + + if partial { + if calledFromFIDsByGID { + epno = test + adb.Logger.Printf("UDP!!! FID %d is only part of episode %s with no complementary files", fid, epno) + } else if len(test) == 1 && test[0].Start.Number == test[0].End.Number { + fids := []int{} + + for fid := range adb.FIDsByGID(thisEp, gid) { + fids = append(fids, int(fid)) + } + if len(fids) >= 1 && fids[0] == 0 { + fids = fids[1:] + // Only entry was API error + if len(fids) == 0 { + return false + } + } + sort.Sort(sort.IntSlice(fids)) + idx := sort.SearchInts(fids, int(fid)) + if idx == len(fids) { + panic(fmt.Sprintf("FID %d couldn't locate itself", fid)) + } + + epno = test + + // equate pointers + epno[0].End = epno[0].Start + + epno[0].Start.Parts = len(fids) + epno[0].Start.Part = idx + } else { + panic(fmt.Sprintf("Don't know what to do with partial episode %s (EID %d)", test, eid)) + } + } else { + // if they're all in sequence, then we'll only have a single range in the list + if len(test) == 1 { + epno = test + } else { + // use only the primary epno then + epno = misc.ParseEpisodeList(thisEp.Episode.String()) + } + } + } + } + + epstr := epno.String() + if len(epno) == 1 && epno[0].Type == misc.EpisodeTypeCredits && epno[0].Len() == 1 { + typ := "" + n := 0 + + if ep := <-adb.EpisodeByID(eid); ep == nil { + } else if m := opedRE.FindStringSubmatch(ep.Titles["en"]); len(m) > 2 { + num, err := strconv.ParseInt(m[2], 10, 32) + if err == nil { + n = int(num) + } + + typ = m[1] + } + + gobi := fmt.Sprintf("%d", n) + if n == 0 { + gobi = "" + } + + switch typ { + case "Opening": + epstr = "OP" + gobi + case "Ending": + epstr = "ED" + gobi + } + } version := FileVersion(1) - switch i := ints[6]; { - case i&fileStateV5 != 0: + switch i := ints[7]; { + case i&stateV5 != 0: version = 5 - case i&fileStateV4 != 0: + case i&stateV4 != 0: version = 4 - case i&fileStateV3 != 0: + case i&stateV3 != 0: version = 3 - case i&fileStateV2 != 0: + case i&stateV2 != 0: version = 2 } - // codecs (parts[13]), bitrates (ints[14]), langs (parts[19]) - codecs := strings.Split(parts[13], "'") - bitrates := strings.Split(parts[14], "'") - alangs := strings.Split(parts[19], "'") + codecs := strings.Split(parts[14], "'") + bitrates := strings.Split(parts[15], "'") + alangs := strings.Split(parts[20], "'") streams := make([]AudioStream, len(codecs)) for i := range streams { br, _ := strconv.ParseInt(bitrates[i], 10, 32) @@ -239,57 +395,72 @@ func parseFileResponse(reply udpapi.APIReply) *File { } } - sl := strings.Split(parts[20], "'") + sl := strings.Split(parts[21], "'") slangs := make([]Language, len(sl)) for i := range sl { slangs[i] = Language(sl[i]) } - depth := int(ints[11]) + depth := int(ints[12]) if depth == 0 { depth = 8 } - res := strings.Split(parts[17], "x") + res := strings.Split(parts[18], "x") width, _ := strconv.ParseInt(res[0], 10, 32) height, _ := strconv.ParseInt(res[1], 10, 32) video := VideoInfo{ - Bitrate: int(ints[16]), - Codec: sanitizeCodec(parts[15]), + Bitrate: int(ints[17]), + Codec: sanitizeCodec(parts[16]), ColorDepth: depth, Resolution: image.Rect(0, 0, int(width), int(height)), } - return &File{ - FID: FID(ints[0]), + lidMap := LIDMap{} + if *f != nil { + lidMap = (*f).LID + } + + uid := <-uidChan + if uid != 0 { + lidMap[uid] = lid + } + + *f = &File{ + FID: fid, + + AID: aid, + EID: eid, + GID: gid, + LID: lidMap, - AID: AID(ints[1]), - EID: EID(ints[2]), - GID: GID(ints[3]), + EpisodeString: epstr, + EpisodeNumber: epno, - OtherEpisodes: misc.ParseEpisodeList(parts[4]).Simplify(), - Deprecated: ints[5] != 0, + RelatedEpisodes: related, + Deprecated: ints[6] != 0, - CRCMatch: ints[6]&fileStateCRCOK != 0, - BadCRC: ints[6]&fileStateCRCERR != 0, + CRCMatch: ints[7]&stateCRCOK != 0, + BadCRC: ints[7]&stateCRCERR != 0, Version: version, - Uncensored: ints[6]&fileStateUncensored != 0, - Censored: ints[6]&fileStateCensored != 0, + Uncensored: ints[7]&stateUncensored != 0, + Censored: ints[7]&stateCensored != 0, Incomplete: video.Resolution.Empty(), - Filesize: ints[7], - Ed2kHash: parts[8], - SHA1Hash: parts[9], - CRC32: parts[10], + Filesize: ints[8], + Ed2kHash: parts[9], + SHA1Hash: parts[10], + CRC32: parts[11], - Source: FileSource(parts[12]), + Source: FileSource(parts[13]), AudioStreams: streams, SubtitleLanguages: slangs, VideoInfo: video, - FileExtension: parts[18], + FileExtension: parts[19], - Length: time.Duration(ints[21]) * time.Second, - AirDate: time.Unix(ints[22], 0), + Length: time.Duration(ints[22]) * time.Second, + AirDate: time.Unix(ints[23], 0), } + return true }