filecache.go

   1 package anidb
   2
   3 import (
   4         "encoding/gob"
   5         "fmt"
   6         "github.com/Kovensky/go-anidb/misc"
   7         "github.com/Kovensky/go-anidb/udp"
   8         "image"
   9         "log"
  10         "regexp"
  11         "sort"
  12         "strconv"
  13         "strings"
  14         "time"
  15 )
  16
  17 func init() {
  18         gob.RegisterName("*github.com/Kovensky/go-anidb.File", &File{})
  19         gob.RegisterName("*github.com/Kovensky/go-anidb.fidCache", &fidCache{})
  20         gob.RegisterName("github.com/Kovensky/go-anidb.FID", FID(0))
  21 }
  22
  23 func (f *File) Touch() {
  24         f.Cached = time.Now()
  25 }
  26
  27 func (f *File) IsStale() bool {
  28         if f == nil {
  29                 return true
  30         }
  31         if f.Incomplete {
  32                 return time.Now().Sub(f.Cached) > FileIncompleteCacheDuration
  33         }
  34         return time.Now().Sub(f.Cached) > FileCacheDuration
  35 }
  36
  37 type FID int
  38
  39 // make FID Cacheable
  40
  41 func (e FID) Touch()        {}
  42 func (e FID) IsStale() bool { return false }
  43
  44 func (fid FID) File() *File {
  45         var f File
  46         if cache.Get(&f, "fid", fid) == nil {
  47                 return &f
  48         }
  49         return nil
  50 }
  51
  52 type fidCache struct {
  53         FID
  54         Time time.Time
  55 }
  56
  57 func (c *fidCache) Touch() {
  58         c.Time = time.Now()
  59 }
  60
  61 func (c *fidCache) IsStale() bool {
  62         return time.Now().Sub(c.Time) > FileCacheDuration
  63 }
  64
  65 // Prefetches the Anime, Episode and Group that this
  66 // file is linked to using the given AniDB instance.
  67 //
  68 // Returns a channel where this file will be sent to
  69 // when the prefetching is done; if the file is nil,
  70 // the channel will return nil.
  71 func (f *File) Prefetch(adb *AniDB) <-chan *File {
  72         ch := make(chan *File, 1)
  73         go func() {
  74                 if f != nil {
  75                         a := adb.AnimeByID(f.AID)
  76                         g := adb.GroupByID(f.GID)
  77                         <-a
  78                         <-g
  79                         ch <- f
  80                 }
  81                 close(ch)
  82         }()
  83         return ch
  84 }
  85
  86 // Retrieves a File by its FID. Uses the UDP API.
  87 func (adb *AniDB) FileByID(fid FID) <-chan *File {
  88         keys := []cacheKey{"fid", fid}
  89
  90         ch := make(chan *File, 1)
  91
  92         if fid < 1 {
  93                 ch <- nil
  94                 close(ch)
  95                 return ch
  96         }
  97
  98         ic := make(chan Cacheable, 1)
  99         go func() { ch <- (<-ic).(*File); close(ch) }()
 100         if intentMap.Intent(ic, keys...) {
 101                 return ch
 102         }
 103
 104         if !cache.CheckValid(keys...) {
 105                 intentMap.NotifyClose((*File)(nil), keys...)
 106                 return ch
 107         }
 108
 109         f := fid.File()
 110         if !f.IsStale() {
 111                 intentMap.NotifyClose(f, keys...)
 112                 return ch
 113         }
 114
 115         go func() {
 116                 reply := <-adb.udp.SendRecv("FILE",
 117                         paramMap{
 118                                 "fid":   fid,
 119                                 "fmask": fileFmask,
 120                                 "amask": fileAmask,
 121                         })
 122
 123                 if reply.Error() == nil {
 124                         f = adb.parseFileResponse(reply, false)
 125
 126                         cache.Set(&fidCache{FID: f.FID}, "fid", "by-ed2k", f.Ed2kHash, f.Filesize)
 127                         cache.Set(f, keys...)
 128                 } else if reply.Code() == 320 {
 129                         cache.MarkInvalid(keys...)
 130                 }
 131
 132                 intentMap.NotifyClose(f, keys...)
 133         }()
 134         return ch
 135 }
 136
 137 var validEd2kHash = regexp.MustCompile(`\A[:xdigit:]{32}\z`)
 138
 139 // Retrieves a File by its Ed2kHash + Filesize combination. Uses the UDP API.
 140 func (adb *AniDB) FileByEd2kSize(ed2k string, size int64) <-chan *File {
 141         keys := []cacheKey{"fid", "by-ed2k", ed2k, size}
 142
 143         ch := make(chan *File, 1)
 144
 145         if size < 1 || !validEd2kHash.MatchString(ed2k) {
 146                 ch <- nil
 147                 close(ch)
 148                 return ch
 149         }
 150         // AniDB always uses lower case hashes
 151         ed2k = strings.ToLower(ed2k)
 152
 153         ic := make(chan Cacheable, 1)
 154         go func() {
 155                 fid := (<-ic).(FID)
 156                 if fid > 0 {
 157                         ch <- <-adb.FileByID(fid)
 158                 }
 159                 close(ch)
 160         }()
 161         if intentMap.Intent(ic, keys...) {
 162                 return ch
 163         }
 164
 165         if !cache.CheckValid(keys...) {
 166                 intentMap.NotifyClose(FID(0), keys...)
 167                 return ch
 168         }
 169
 170         fid := FID(0)
 171
 172         var ec fidCache
 173         if cache.Get(&ec, keys...) == nil && !ec.IsStale() {
 174                 intentMap.NotifyClose(ec.FID, keys...)
 175                 return ch
 176         }
 177         fid = ec.FID
 178
 179         go func() {
 180                 reply := <-adb.udp.SendRecv("FILE",
 181                         paramMap{
 182                                 "ed2k":  ed2k,
 183                                 "size":  size,
 184                                 "fmask": fileFmask,
 185                                 "amask": fileAmask,
 186                         })
 187
 188                 var f *File
 189                 if reply.Error() == nil {
 190                         f = adb.parseFileResponse(reply, false)
 191
 192                         fid = f.FID
 193
 194                         cache.Set(&fidCache{FID: fid}, keys...)
 195                         cache.Set(f, "fid", fid)
 196                 } else if reply.Code() == 320 { // file not found
 197                         cache.MarkInvalid(keys...)
 198                 } else if reply.Code() == 322 { // multiple files found
 199                         panic("Don't know what to do with " + strings.Join(reply.Lines(), "\n"))
 200                 }
 201
 202                 intentMap.NotifyClose(fid, keys...)
 203         }()
 204         return ch
 205 }
 206
 207 var fileFmask = "77da7fe8"
 208 var fileAmask = "00008000"
 209
 210 const (
 211         fileStateCRCOK = 1 << iota
 212         fileStateCRCERR
 213         fileStateV2
 214         fileStateV3
 215         fileStateV4
 216         fileStateV5
 217         fileStateUncensored
 218         fileStateCensored
 219 )
 220
 221 func sanitizeCodec(codec string) string {
 222         switch codec {
 223         case "MP3 CBR":
 224                 return "MP3"
 225         case "WMV9 (also WMV3)":
 226                 return "WMV9"
 227         case "Ogg (Vorbis)":
 228                 return "Vorbis"
 229         case "H264/AVC":
 230                 return "H.264"
 231         }
 232         return codec
 233 }
 234
 235 func (adb *AniDB) parseFileResponse(reply udpapi.APIReply, calledFromFIDsByGID bool) *File {
 236         if reply.Error() != nil {
 237                 return nil
 238         }
 239         if reply.Truncated() {
 240                 panic("Truncated")
 241         }
 242
 243         parts := strings.Split(reply.Lines()[1], "|")
 244         ints := make([]int64, len(parts))
 245         for i, p := range parts {
 246                 ints[i], _ = strconv.ParseInt(p, 10, 64)
 247         }
 248
 249         partial := false
 250
 251         rels := strings.Split(parts[4], "'")
 252         relList := make([]EID, 0, len(parts[4]))
 253         related := make(RelatedEpisodes, len(parts[4]))
 254         for _, rel := range rels {
 255                 r := strings.Split(rel, ",")
 256                 if len(r) < 2 {
 257                         continue
 258                 }
 259
 260                 eid, _ := strconv.ParseInt(r[0], 10, 32)
 261                 pct, _ := strconv.ParseInt(r[1], 10, 32)
 262                 relList = append(relList, EID(eid))
 263                 related[EID(eid)] = float32(pct) / 100
 264
 265                 if pct != 100 {
 266                         partial = true
 267                 }
 268         }
 269
 270         epno := misc.ParseEpisodeList(parts[23])
 271         fid := FID(ints[0])
 272         aid := AID(ints[1])
 273         eid := EID(ints[2])
 274         gid := GID(ints[3])
 275
 276         if !epno[0].Start.ContainsEpisodes(epno[0].End) || len(epno) > 1 || len(relList) > 0 {
 277                 // epno is broken -- we need to sanitize it
 278                 thisEp := <-adb.EpisodeByID(eid)
 279                 bad := false
 280                 if thisEp != nil {
 281                         parts := make([]string, 1, len(relList)+1)
 282                         parts[0] = thisEp.Episode.String()
 283
 284                         // everything after this SHOULD be cache hits now, unless this is somehow
 285                         // linked with an EID from a different anime (*stares at Haruhi*).
 286                         // We don't want to use eps from different AIDs anyway, so that makes
 287                         // the job easier.
 288
 289                         // We check if the related episodes are all in sequence from this one.
 290                         // If they are, we build a new epno with the sequence. Otherwise,
 291                         // our epno will only have the primary episode.
 292
 293                         // gather the episode numbers
 294                         for _, eid := range relList {
 295                                 if ep := eid.Episode(); ep != nil && ep.AID == thisEp.AID {
 296                                         parts = append(parts, ep.Episode.String())
 297                                 } else {
 298                                         bad = true
 299                                         break
 300                                 }
 301                         }
 302
 303                         test := misc.EpisodeList{}
 304                         // only if we didn't break the loop
 305                         if !bad {
 306                                 test = misc.ParseEpisodeList(strings.Join(parts, ","))
 307                         }
 308
 309                         if partial {
 310                                 if calledFromFIDsByGID {
 311                                         epno = test
 312                                         log.Printf("UDP!!! FID %d is only part of episode %s with no complementary files", fid, epno)
 313                                 } else if len(test) == 1 && test[0].Start.Number == test[0].End.Number {
 314                                         fids := []int{}
 315
 316                                         for fid := range adb.FIDsByGID(thisEp, gid) {
 317                                                 fids = append(fids, int(fid))
 318                                         }
 319                                         if len(fids) >= 1 && fids[0] == 0 {
 320                                                 fids = fids[1:]
 321                                                 // Only entry was API error
 322                                                 if len(fids) == 0 {
 323                                                         return nil
 324                                                 }
 325                                         }
 326                                         sort.Sort(sort.IntSlice(fids))
 327                                         idx := sort.SearchInts(fids, int(fid))
 328                                         if idx == len(fids) {
 329                                                 panic(fmt.Sprintf("FID %d couldn't locate itself", fid))
 330                                         }
 331
 332                                         epno = test
 333
 334                                         // equate pointers
 335                                         epno[0].End = epno[0].Start
 336
 337                                         epno[0].Start.Parts = len(fids)
 338                                         epno[0].Start.Part = idx
 339                                 } else {
 340                                         panic(fmt.Sprintf("Don't know what to do with partial episode %s (EID %d)", test, eid))
 341                                 }
 342                         } else {
 343                                 // if they're all in sequence, then we'll only have a single range in the list
 344                                 if len(test) == 1 {
 345                                         epno = test
 346                                 } else {
 347                                         // use only the primary epno then
 348                                         epno = misc.ParseEpisodeList(thisEp.Episode.String())
 349                                 }
 350                         }
 351                 }
 352         }
 353
 354         version := FileVersion(1)
 355         switch i := ints[6]; {
 356         case i&fileStateV5 != 0:
 357                 version = 5
 358         case i&fileStateV4 != 0:
 359                 version = 4
 360         case i&fileStateV3 != 0:
 361                 version = 3
 362         case i&fileStateV2 != 0:
 363                 version = 2
 364         }
 365
 366         // codecs (parts[13]), bitrates (ints[14]), langs (parts[19])
 367         codecs := strings.Split(parts[13], "'")
 368         bitrates := strings.Split(parts[14], "'")
 369         alangs := strings.Split(parts[19], "'")
 370         streams := make([]AudioStream, len(codecs))
 371         for i := range streams {
 372                 br, _ := strconv.ParseInt(bitrates[i], 10, 32)
 373                 streams[i] = AudioStream{
 374                         Bitrate:  int(br),
 375                         Codec:    sanitizeCodec(codecs[i]),
 376                         Language: Language(alangs[i]),
 377                 }
 378         }
 379
 380         sl := strings.Split(parts[20], "'")
 381         slangs := make([]Language, len(sl))
 382         for i := range sl {
 383                 slangs[i] = Language(sl[i])
 384         }
 385
 386         depth := int(ints[11])
 387         if depth == 0 {
 388                 depth = 8
 389         }
 390         res := strings.Split(parts[17], "x")
 391         width, _ := strconv.ParseInt(res[0], 10, 32)
 392         height, _ := strconv.ParseInt(res[1], 10, 32)
 393         video := VideoInfo{
 394                 Bitrate:    int(ints[16]),
 395                 Codec:      sanitizeCodec(parts[15]),
 396                 ColorDepth: depth,
 397                 Resolution: image.Rect(0, 0, int(width), int(height)),
 398         }
 399
 400         return &File{
 401                 FID: fid,
 402
 403                 AID: aid,
 404                 EID: eid,
 405                 GID: gid,
 406
 407                 EpisodeNumber: epno,
 408
 409                 RelatedEpisodes: related,
 410                 Deprecated:      ints[5] != 0,
 411
 412                 CRCMatch:   ints[6]&fileStateCRCOK != 0,
 413                 BadCRC:     ints[6]&fileStateCRCERR != 0,
 414                 Version:    version,
 415                 Uncensored: ints[6]&fileStateUncensored != 0,
 416                 Censored:   ints[6]&fileStateCensored != 0,
 417
 418                 Incomplete: video.Resolution.Empty(),
 419
 420                 Filesize: ints[7],
 421                 Ed2kHash: parts[8],
 422                 SHA1Hash: parts[9],
 423                 CRC32:    parts[10],
 424
 425                 Source: FileSource(parts[12]),
 426
 427                 AudioStreams:      streams,
 428                 SubtitleLanguages: slangs,
 429                 VideoInfo:         video,
 430                 FileExtension:     parts[18],
 431
 432                 Length:  time.Duration(ints[21]) * time.Second,
 433                 AirDate: time.Unix(ints[22], 0),
 434         }
 435 }