filecache.go

   1 package anidb
   2
   3 import (
   4         "fmt"
   5         "github.com/Kovensky/go-anidb/misc"
   6         "github.com/Kovensky/go-anidb/udp"
   7         "github.com/Kovensky/go-fscache"
   8         "image"
   9         "log"
  10         "regexp"
  11         "sort"
  12         "strconv"
  13         "strings"
  14         "time"
  15 )
  16
  17 var _ cacheable = &File{}
  18
  19 func (f *File) setCachedTS(ts time.Time) {
  20         f.Cached = ts
  21 }
  22
  23 func (f *File) IsStale() bool {
  24         if f == nil {
  25                 return true
  26         }
  27         if f.Incomplete {
  28                 return time.Now().Sub(f.Cached) > FileIncompleteCacheDuration
  29         }
  30         return time.Now().Sub(f.Cached) > FileCacheDuration
  31 }
  32
  33 func cacheFile(f *File) {
  34         CacheSet(f.FID, "fid", "by-ed2k", f.Ed2kHash, f.Filesize)
  35         CacheSet(f, "fid", f.FID)
  36 }
  37
  38 type FID int
  39
  40 func (fid FID) File() *File {
  41         var f File
  42         if CacheGet(&f, "fid", fid) == nil {
  43                 return &f
  44         }
  45         return nil
  46 }
  47
  48 // Prefetches the Anime, Episode and Group that this
  49 // file is linked to using the given AniDB instance.
  50 //
  51 // Returns a channel where this file will be sent to
  52 // when the prefetching is done; if the file is nil,
  53 // the channel will return nil.
  54 func (f *File) Prefetch(adb *AniDB) <-chan *File {
  55         ch := make(chan *File, 1)
  56         go func() {
  57                 if f != nil {
  58                         a := adb.AnimeByID(f.AID)
  59                         g := adb.GroupByID(f.GID)
  60                         <-a
  61                         <-g
  62                         ch <- f
  63                 }
  64                 close(ch)
  65         }()
  66         return ch
  67 }
  68
  69 // Retrieves a File by its FID. Uses the UDP API.
  70 func (adb *AniDB) FileByID(fid FID) <-chan *File {
  71         key := []fscache.CacheKey{"fid", fid}
  72
  73         ch := make(chan *File, 1)
  74
  75         if fid < 1 {
  76                 ch <- nil
  77                 close(ch)
  78                 return ch
  79         }
  80
  81         ic := make(chan notification, 1)
  82         go func() { ch <- (<-ic).(*File); close(ch) }()
  83         if intentMap.Intent(ic, key...) {
  84                 return ch
  85         }
  86
  87         if !Cache.IsValid(InvalidKeyCacheDuration, key...) {
  88                 intentMap.NotifyClose((*File)(nil), key...)
  89                 return ch
  90         }
  91
  92         f := fid.File()
  93         if !f.IsStale() {
  94                 intentMap.NotifyClose(f, key...)
  95                 return ch
  96         }
  97
  98         go func() {
  99                 reply := <-adb.udp.SendRecv("FILE",
 100                         paramMap{
 101                                 "fid":   fid,
 102                                 "fmask": fileFmask,
 103                                 "amask": fileAmask,
 104                         })
 105
 106                 if reply.Error() == nil {
 107                         adb.parseFileResponse(&f, reply, false)
 108
 109                         cacheFile(f)
 110                 } else if reply.Code() == 320 {
 111                         Cache.SetInvalid(key...)
 112                 }
 113
 114                 intentMap.NotifyClose(f, key...)
 115         }()
 116         return ch
 117 }
 118
 119 var validEd2kHash = regexp.MustCompile(`\A[:xdigit:]{32}\z`)
 120
 121 // Retrieves a File by its Ed2kHash + Filesize combination. Uses the UDP API.
 122 func (adb *AniDB) FileByEd2kSize(ed2k string, size int64) <-chan *File {
 123         key := []fscache.CacheKey{"fid", "by-ed2k", ed2k, size}
 124
 125         ch := make(chan *File, 1)
 126
 127         if size < 1 || !validEd2kHash.MatchString(ed2k) {
 128                 ch <- nil
 129                 close(ch)
 130                 return ch
 131         }
 132         // AniDB always uses lower case hashes
 133         ed2k = strings.ToLower(ed2k)
 134
 135         ic := make(chan notification, 1)
 136         go func() {
 137                 fid := (<-ic).(FID)
 138                 if fid > 0 {
 139                         ch <- <-adb.FileByID(fid)
 140                 }
 141                 close(ch)
 142         }()
 143         if intentMap.Intent(ic, key...) {
 144                 return ch
 145         }
 146
 147         if !Cache.IsValid(InvalidKeyCacheDuration, key...) {
 148                 intentMap.NotifyClose(FID(0), key...)
 149                 return ch
 150         }
 151
 152         fid := FID(0)
 153
 154         switch ts, err := Cache.Get(&fid, key...); {
 155         case err != nil && time.Now().Sub(ts) < FileCacheDuration:
 156                 intentMap.NotifyClose(fid, key...)
 157                 return ch
 158         }
 159
 160         go func() {
 161                 reply := <-adb.udp.SendRecv("FILE",
 162                         paramMap{
 163                                 "ed2k":  ed2k,
 164                                 "size":  size,
 165                                 "fmask": fileFmask,
 166                                 "amask": fileAmask,
 167                         })
 168
 169                 var f *File
 170                 if reply.Error() == nil {
 171                         adb.parseFileResponse(&f, reply, false)
 172
 173                         fid = f.FID
 174
 175                         cacheFile(f)
 176                 } else if reply.Code() == 320 { // file not found
 177                         Cache.SetInvalid(key...)
 178                 } else if reply.Code() == 322 { // multiple files found
 179                         panic("Don't know what to do with " + strings.Join(reply.Lines(), "\n"))
 180                 }
 181
 182                 intentMap.NotifyClose(fid, key...)
 183         }()
 184         return ch
 185 }
 186
 187 var fileFmask = "7fda7fe8"
 188 var fileAmask = "00008000"
 189
 190 const (
 191         stateCRCOK = 1 << iota
 192         stateCRCERR
 193         stateV2
 194         stateV3
 195         stateV4
 196         stateV5
 197         stateUncensored
 198         stateCensored
 199 )
 200
 201 func sanitizeCodec(codec string) string {
 202         switch codec {
 203         case "MP3 CBR":
 204                 return "MP3"
 205         case "WMV9 (also WMV3)":
 206                 return "WMV9"
 207         case "Ogg (Vorbis)":
 208                 return "Vorbis"
 209         case "H264/AVC":
 210                 return "H.264"
 211         }
 212         return codec
 213 }
 214
 215 func (adb *AniDB) parseFileResponse(f **File, reply udpapi.APIReply, calledFromFIDsByGID bool) bool {
 216         if reply.Error() != nil {
 217                 return false
 218         }
 219         if reply.Truncated() {
 220                 panic("Truncated")
 221         }
 222
 223         uidChan := make(chan UID, 1)
 224         if adb.udp.credentials != nil {
 225                 go func() { uidChan <- <-adb.GetUserUID(decrypt(adb.udp.credentials.username)) }()
 226         } else {
 227                 uidChan <- 0
 228                 close(uidChan)
 229         }
 230
 231         parts := strings.Split(reply.Lines()[1], "|")
 232         ints := make([]int64, len(parts))
 233         for i, p := range parts {
 234                 ints[i], _ = strconv.ParseInt(p, 10, 64)
 235         }
 236
 237         partial := false
 238
 239         rels := strings.Split(parts[5], "'")
 240         relList := make([]EID, 0, len(parts[5]))
 241         related := make(RelatedEpisodes, len(parts[5]))
 242         for _, rel := range rels {
 243                 r := strings.Split(rel, ",")
 244                 if len(r) < 2 {
 245                         continue
 246                 }
 247
 248                 eid, _ := strconv.ParseInt(r[0], 10, 32)
 249                 pct, _ := strconv.ParseInt(r[1], 10, 32)
 250                 relList = append(relList, EID(eid))
 251                 related[EID(eid)] = float32(pct) / 100
 252
 253                 if pct != 100 {
 254                         partial = true
 255                 }
 256         }
 257
 258         epno := misc.ParseEpisodeList(parts[24])
 259         fid := FID(ints[0])
 260         aid := AID(ints[1])
 261         eid := EID(ints[2])
 262         gid := GID(ints[3])
 263         lid := LID(ints[4])
 264
 265         if !epno[0].Start.ContainsEpisodes(epno[0].End) || len(epno) > 1 || len(relList) > 0 {
 266                 // epno is broken -- we need to sanitize it
 267                 thisEp := <-adb.EpisodeByID(eid)
 268                 bad := false
 269                 if thisEp != nil {
 270                         parts := make([]string, 1, len(relList)+1)
 271                         parts[0] = thisEp.Episode.String()
 272
 273                         // everything after this SHOULD be cache hits now, unless this is somehow
 274                         // linked with an EID from a different anime (*stares at Haruhi*).
 275                         // We don't want to use eps from different AIDs anyway, so that makes
 276                         // the job easier.
 277
 278                         // We check if the related episodes are all in sequence from this one.
 279                         // If they are, we build a new epno with the sequence. Otherwise,
 280                         // our epno will only have the primary episode.
 281
 282                         // gather the episode numbers
 283                         for _, eid := range relList {
 284                                 if ep := eid.Episode(); ep != nil && ep.AID == thisEp.AID {
 285                                         parts = append(parts, ep.Episode.String())
 286                                 } else {
 287                                         bad = true
 288                                         break
 289                                 }
 290                         }
 291
 292                         test := misc.EpisodeList{}
 293                         // only if we didn't break the loop
 294                         if !bad {
 295                                 test = misc.ParseEpisodeList(strings.Join(parts, ","))
 296                         }
 297
 298                         if partial {
 299                                 if calledFromFIDsByGID {
 300                                         epno = test
 301                                         log.Printf("UDP!!! FID %d is only part of episode %s with no complementary files", fid, epno)
 302                                 } else if len(test) == 1 && test[0].Start.Number == test[0].End.Number {
 303                                         fids := []int{}
 304
 305                                         for fid := range adb.FIDsByGID(thisEp, gid) {
 306                                                 fids = append(fids, int(fid))
 307                                         }
 308                                         if len(fids) >= 1 && fids[0] == 0 {
 309                                                 fids = fids[1:]
 310                                                 // Only entry was API error
 311                                                 if len(fids) == 0 {
 312                                                         return false
 313                                                 }
 314                                         }
 315                                         sort.Sort(sort.IntSlice(fids))
 316                                         idx := sort.SearchInts(fids, int(fid))
 317                                         if idx == len(fids) {
 318                                                 panic(fmt.Sprintf("FID %d couldn't locate itself", fid))
 319                                         }
 320
 321                                         epno = test
 322
 323                                         // equate pointers
 324                                         epno[0].End = epno[0].Start
 325
 326                                         epno[0].Start.Parts = len(fids)
 327                                         epno[0].Start.Part = idx
 328                                 } else {
 329                                         panic(fmt.Sprintf("Don't know what to do with partial episode %s (EID %d)", test, eid))
 330                                 }
 331                         } else {
 332                                 // if they're all in sequence, then we'll only have a single range in the list
 333                                 if len(test) == 1 {
 334                                         epno = test
 335                                 } else {
 336                                         // use only the primary epno then
 337                                         epno = misc.ParseEpisodeList(thisEp.Episode.String())
 338                                 }
 339                         }
 340                 }
 341         }
 342
 343         version := FileVersion(1)
 344         switch i := ints[7]; {
 345         case i&stateV5 != 0:
 346                 version = 5
 347         case i&stateV4 != 0:
 348                 version = 4
 349         case i&stateV3 != 0:
 350                 version = 3
 351         case i&stateV2 != 0:
 352                 version = 2
 353         }
 354
 355         codecs := strings.Split(parts[14], "'")
 356         bitrates := strings.Split(parts[15], "'")
 357         alangs := strings.Split(parts[20], "'")
 358         streams := make([]AudioStream, len(codecs))
 359         for i := range streams {
 360                 br, _ := strconv.ParseInt(bitrates[i], 10, 32)
 361                 streams[i] = AudioStream{
 362                         Bitrate:  int(br),
 363                         Codec:    sanitizeCodec(codecs[i]),
 364                         Language: Language(alangs[i]),
 365                 }
 366         }
 367
 368         sl := strings.Split(parts[21], "'")
 369         slangs := make([]Language, len(sl))
 370         for i := range sl {
 371                 slangs[i] = Language(sl[i])
 372         }
 373
 374         depth := int(ints[12])
 375         if depth == 0 {
 376                 depth = 8
 377         }
 378         res := strings.Split(parts[18], "x")
 379         width, _ := strconv.ParseInt(res[0], 10, 32)
 380         height, _ := strconv.ParseInt(res[1], 10, 32)
 381         video := VideoInfo{
 382                 Bitrate:    int(ints[17]),
 383                 Codec:      sanitizeCodec(parts[16]),
 384                 ColorDepth: depth,
 385                 Resolution: image.Rect(0, 0, int(width), int(height)),
 386         }
 387
 388         lidMap := LIDMap{}
 389         if *f != nil {
 390                 lidMap = (*f).LID
 391         }
 392
 393         uid := <-uidChan
 394         if uid != 0 {
 395                 lidMap[uid] = lid
 396         }
 397
 398         *f = &File{
 399                 FID: fid,
 400
 401                 AID: aid,
 402                 EID: eid,
 403                 GID: gid,
 404                 LID: lidMap,
 405
 406                 EpisodeNumber: epno,
 407
 408                 RelatedEpisodes: related,
 409                 Deprecated:      ints[6] != 0,
 410
 411                 CRCMatch:   ints[7]&stateCRCOK != 0,
 412                 BadCRC:     ints[7]&stateCRCERR != 0,
 413                 Version:    version,
 414                 Uncensored: ints[7]&stateUncensored != 0,
 415                 Censored:   ints[7]&stateCensored != 0,
 416
 417                 Incomplete: video.Resolution.Empty(),
 418
 419                 Filesize: ints[8],
 420                 Ed2kHash: parts[9],
 421                 SHA1Hash: parts[10],
 422                 CRC32:    parts[11],
 423
 424                 Source: FileSource(parts[13]),
 425
 426                 AudioStreams:      streams,
 427                 SubtitleLanguages: slangs,
 428                 VideoInfo:         video,
 429                 FileExtension:     parts[19],
 430
 431                 Length:  time.Duration(ints[22]) * time.Second,
 432                 AirDate: time.Unix(ints[23], 0),
 433         }
 434         return true
 435 }