filecache.go

   1 package anidb
   2
   3 import (
   4         "fmt"
   5         "github.com/Kovensky/go-anidb/misc"
   6         "github.com/Kovensky/go-anidb/udp"
   7         "github.com/Kovensky/go-fscache"
   8         "image"
   9         "regexp"
  10         "sort"
  11         "strconv"
  12         "strings"
  13         "time"
  14 )
  15
  16 var _ cacheable = &File{}
  17
  18 func (f *File) setCachedTS(ts time.Time) {
  19         f.Cached = ts
  20 }
  21
  22 func (f *File) IsStale() bool {
  23         if f == nil {
  24                 return true
  25         }
  26         if f.Incomplete {
  27                 return time.Now().Sub(f.Cached) > FileIncompleteCacheDuration
  28         }
  29         return time.Now().Sub(f.Cached) > FileCacheDuration
  30 }
  31
  32 func cacheFile(f *File) {
  33         CacheSet(f.FID, "fid", "by-ed2k", f.Ed2kHash, f.Filesize)
  34         CacheSet(f, "fid", f.FID)
  35 }
  36
  37 type FID int
  38
  39 func (fid FID) File() *File {
  40         var f File
  41         if CacheGet(&f, "fid", fid) == nil {
  42                 return &f
  43         }
  44         return nil
  45 }
  46
  47 // Prefetches the Anime, Episode and Group that this
  48 // file is linked to using the given AniDB instance.
  49 //
  50 // Returns a channel where this file will be sent to
  51 // when the prefetching is done; if the file is nil,
  52 // the channel will return nil.
  53 func (f *File) Prefetch(adb *AniDB) <-chan *File {
  54         ch := make(chan *File, 1)
  55         go func() {
  56                 if f != nil {
  57                         a := adb.AnimeByID(f.AID)
  58                         g := adb.GroupByID(f.GID)
  59                         <-a
  60                         <-g
  61                         ch <- f
  62                 }
  63                 close(ch)
  64         }()
  65         return ch
  66 }
  67
  68 // Retrieves a File by its FID. Uses the UDP API.
  69 func (adb *AniDB) FileByID(fid FID) <-chan *File {
  70         key := []fscache.CacheKey{"fid", fid}
  71
  72         ch := make(chan *File, 1)
  73
  74         if fid < 1 {
  75                 ch <- nil
  76                 close(ch)
  77                 return ch
  78         }
  79
  80         ic := make(chan notification, 1)
  81         go func() { ch <- (<-ic).(*File); close(ch) }()
  82         if intentMap.Intent(ic, key...) {
  83                 return ch
  84         }
  85
  86         if !Cache.IsValid(InvalidKeyCacheDuration, key...) {
  87                 intentMap.NotifyClose((*File)(nil), key...)
  88                 return ch
  89         }
  90
  91         f := fid.File()
  92         if !f.IsStale() {
  93                 intentMap.NotifyClose(f, key...)
  94                 return ch
  95         }
  96
  97         go func() {
  98                 reply := <-adb.udp.SendRecv("FILE",
  99                         paramMap{
 100                                 "fid":   fid,
 101                                 "fmask": fileFmask,
 102                                 "amask": fileAmask,
 103                         })
 104
 105                 if reply.Error() == nil {
 106                         adb.parseFileResponse(&f, reply, false)
 107
 108                         cacheFile(f)
 109                 } else if reply.Code() == 320 {
 110                         Cache.SetInvalid(key...)
 111                 }
 112
 113                 intentMap.NotifyClose(f, key...)
 114         }()
 115         return ch
 116 }
 117
 118 var validEd2kHash = regexp.MustCompile(`\A[[:xdigit:]]{32}\z`)
 119
 120 // Retrieves a File by its Ed2kHash + Filesize combination. Uses the UDP API.
 121 func (adb *AniDB) FileByEd2kSize(ed2k string, size int64) <-chan *File {
 122         key := []fscache.CacheKey{"fid", "by-ed2k", ed2k, size}
 123
 124         ch := make(chan *File, 1)
 125
 126         if size < 1 || !validEd2kHash.MatchString(ed2k) {
 127                 ch <- nil
 128                 close(ch)
 129                 return ch
 130         }
 131         // AniDB always uses lower case hashes
 132         ed2k = strings.ToLower(ed2k)
 133
 134         ic := make(chan notification, 1)
 135         go func() {
 136                 fid := (<-ic).(FID)
 137                 if fid > 0 {
 138                         ch <- <-adb.FileByID(fid)
 139                 }
 140                 close(ch)
 141         }()
 142         if intentMap.Intent(ic, key...) {
 143                 return ch
 144         }
 145
 146         if !Cache.IsValid(InvalidKeyCacheDuration, key...) {
 147                 intentMap.NotifyClose(FID(0), key...)
 148                 return ch
 149         }
 150
 151         fid := FID(0)
 152
 153         switch ts, err := Cache.Get(&fid, key...); {
 154         case err == nil && time.Now().Sub(ts) < FileCacheDuration:
 155                 intentMap.NotifyClose(fid, key...)
 156                 return ch
 157         }
 158
 159         go func() {
 160                 reply := <-adb.udp.SendRecv("FILE",
 161                         paramMap{
 162                                 "ed2k":  ed2k,
 163                                 "size":  size,
 164                                 "fmask": fileFmask,
 165                                 "amask": fileAmask,
 166                         })
 167
 168                 var f *File
 169                 if reply.Error() == nil {
 170                         adb.parseFileResponse(&f, reply, false)
 171
 172                         fid = f.FID
 173
 174                         cacheFile(f)
 175                 } else if reply.Code() == 320 { // file not found
 176                         Cache.SetInvalid(key...)
 177                 } else if reply.Code() == 322 { // multiple files found
 178                         panic("Don't know what to do with " + strings.Join(reply.Lines(), "\n"))
 179                 }
 180
 181                 intentMap.NotifyClose(fid, key...)
 182         }()
 183         return ch
 184 }
 185
 186 var fileFmask = "7fda7fe8"
 187 var fileAmask = "00008000"
 188
 189 const (
 190         stateCRCOK = 1 << iota
 191         stateCRCERR
 192         stateV2
 193         stateV3
 194         stateV4
 195         stateV5
 196         stateUncensored
 197         stateCensored
 198 )
 199
 200 func sanitizeCodec(codec string) string {
 201         switch codec {
 202         case "MP3 CBR":
 203                 return "MP3"
 204         case "WMV9 (also WMV3)":
 205                 return "WMV9"
 206         case "Ogg (Vorbis)":
 207                 return "Vorbis"
 208         case "H264/AVC":
 209                 return "H.264"
 210         }
 211         return codec
 212 }
 213
 214 var opedRE = regexp.MustCompile(`\A(Opening|Ending)(?: (\d+))?\z`)
 215
 216 func (adb *AniDB) parseFileResponse(f **File, reply udpapi.APIReply, calledFromFIDsByGID bool) bool {
 217         if reply.Error() != nil {
 218                 return false
 219         }
 220         if reply.Truncated() {
 221                 panic("Truncated")
 222         }
 223
 224         uidChan := make(chan UID, 1)
 225         if adb.udp.credentials != nil {
 226                 go func() { uidChan <- <-adb.GetUserUID(decrypt(adb.udp.credentials.username)) }()
 227         } else {
 228                 uidChan <- 0
 229                 close(uidChan)
 230         }
 231
 232         parts := strings.Split(reply.Lines()[1], "|")
 233         ints := make([]int64, len(parts))
 234         for i, p := range parts {
 235                 ints[i], _ = strconv.ParseInt(p, 10, 64)
 236         }
 237
 238         partial := false
 239
 240         rels := strings.Split(parts[5], "'")
 241         relList := make([]EID, 0, len(parts[5]))
 242         related := make(RelatedEpisodes, len(parts[5]))
 243         for _, rel := range rels {
 244                 r := strings.Split(rel, ",")
 245                 if len(r) < 2 {
 246                         continue
 247                 }
 248
 249                 eid, _ := strconv.ParseInt(r[0], 10, 32)
 250                 pct, _ := strconv.ParseInt(r[1], 10, 32)
 251                 relList = append(relList, EID(eid))
 252                 related[EID(eid)] = float32(pct) / 100
 253
 254                 if pct != 100 {
 255                         partial = true
 256                 }
 257         }
 258
 259         epno := misc.ParseEpisodeList(parts[24])
 260         fid := FID(ints[0])
 261         aid := AID(ints[1])
 262         eid := EID(ints[2])
 263         gid := GID(ints[3])
 264         lid := LID(ints[4])
 265
 266         if !epno[0].Start.ContainsEpisodes(epno[0].End) || len(epno) > 1 || len(relList) > 0 {
 267                 // epno is broken -- we need to sanitize it
 268                 thisEp := <-adb.EpisodeByID(eid)
 269                 bad := false
 270                 if thisEp != nil {
 271                         parts := make([]string, 1, len(relList)+1)
 272                         parts[0] = thisEp.Episode.String()
 273
 274                         // everything after this SHOULD be cache hits now, unless this is somehow
 275                         // linked with an EID from a different anime (*stares at Haruhi*).
 276                         // We don't want to use eps from different AIDs anyway, so that makes
 277                         // the job easier.
 278
 279                         // We check if the related episodes are all in sequence from this one.
 280                         // If they are, we build a new epno with the sequence. Otherwise,
 281                         // our epno will only have the primary episode.
 282
 283                         // gather the episode numbers
 284                         for _, eid := range relList {
 285                                 if ep := eid.Episode(); ep != nil && ep.AID == thisEp.AID {
 286                                         parts = append(parts, ep.Episode.String())
 287                                 } else {
 288                                         bad = true
 289                                         break
 290                                 }
 291                         }
 292
 293                         test := misc.EpisodeList{}
 294                         // only if we didn't break the loop
 295                         if !bad {
 296                                 test = misc.ParseEpisodeList(strings.Join(parts, ","))
 297                         }
 298
 299                         if partial {
 300                                 if calledFromFIDsByGID {
 301                                         epno = test
 302                                         adb.Logger.Printf("UDP!!! FID %d is only part of episode %s with no complementary files", fid, epno)
 303                                 } else if len(test) == 1 && test[0].Start.Number == test[0].End.Number {
 304                                         fids := []int{}
 305
 306                                         for fid := range adb.FIDsByGID(thisEp, gid) {
 307                                                 fids = append(fids, int(fid))
 308                                         }
 309                                         if len(fids) >= 1 && fids[0] == 0 {
 310                                                 fids = fids[1:]
 311                                                 // Only entry was API error
 312                                                 if len(fids) == 0 {
 313                                                         return false
 314                                                 }
 315                                         }
 316                                         sort.Sort(sort.IntSlice(fids))
 317                                         idx := sort.SearchInts(fids, int(fid))
 318                                         if idx == len(fids) {
 319                                                 panic(fmt.Sprintf("FID %d couldn't locate itself", fid))
 320                                         }
 321
 322                                         epno = test
 323
 324                                         // equate pointers
 325                                         epno[0].End = epno[0].Start
 326
 327                                         epno[0].Start.Parts = len(fids)
 328                                         epno[0].Start.Part = idx
 329                                 } else {
 330                                         panic(fmt.Sprintf("Don't know what to do with partial episode %s (EID %d)", test, eid))
 331                                 }
 332                         } else {
 333                                 // if they're all in sequence, then we'll only have a single range in the list
 334                                 if len(test) == 1 {
 335                                         epno = test
 336                                 } else {
 337                                         // use only the primary epno then
 338                                         epno = misc.ParseEpisodeList(thisEp.Episode.String())
 339                                 }
 340                         }
 341                 }
 342         }
 343
 344         epstr := epno.String()
 345         if len(epno) == 1 && epno[0].Type == misc.EpisodeTypeCredits && epno[0].Len() == 1 {
 346                 typ := ""
 347                 n := 0
 348
 349                 if ep := <-adb.EpisodeByID(eid); ep == nil {
 350                 } else if m := opedRE.FindStringSubmatch(ep.Titles["en"]); len(m) > 2 {
 351                         num, err := strconv.ParseInt(m[2], 10, 32)
 352                         if err == nil {
 353                                 n = int(num)
 354                         }
 355
 356                         typ = m[1]
 357                 }
 358
 359                 gobi := fmt.Sprintf("%d", n)
 360                 if n == 0 {
 361                         gobi = ""
 362                 }
 363
 364                 switch typ {
 365                 case "Opening":
 366                         epstr = "OP" + gobi
 367                 case "Ending":
 368                         epstr = "ED" + gobi
 369                 }
 370         }
 371
 372         version := FileVersion(1)
 373         switch i := ints[7]; {
 374         case i&stateV5 != 0:
 375                 version = 5
 376         case i&stateV4 != 0:
 377                 version = 4
 378         case i&stateV3 != 0:
 379                 version = 3
 380         case i&stateV2 != 0:
 381                 version = 2
 382         }
 383
 384         codecs := strings.Split(parts[14], "'")
 385         bitrates := strings.Split(parts[15], "'")
 386         alangs := strings.Split(parts[20], "'")
 387         streams := make([]AudioStream, len(codecs))
 388         for i := range streams {
 389                 br, _ := strconv.ParseInt(bitrates[i], 10, 32)
 390                 streams[i] = AudioStream{
 391                         Bitrate:  int(br),
 392                         Codec:    sanitizeCodec(codecs[i]),
 393                         Language: Language(alangs[i]),
 394                 }
 395         }
 396
 397         sl := strings.Split(parts[21], "'")
 398         slangs := make([]Language, len(sl))
 399         for i := range sl {
 400                 slangs[i] = Language(sl[i])
 401         }
 402
 403         depth := int(ints[12])
 404         if depth == 0 {
 405                 depth = 8
 406         }
 407         res := strings.Split(parts[18], "x")
 408         width, _ := strconv.ParseInt(res[0], 10, 32)
 409         height, _ := strconv.ParseInt(res[1], 10, 32)
 410         video := VideoInfo{
 411                 Bitrate:    int(ints[17]),
 412                 Codec:      sanitizeCodec(parts[16]),
 413                 ColorDepth: depth,
 414                 Resolution: image.Rect(0, 0, int(width), int(height)),
 415         }
 416
 417         lidMap := LIDMap{}
 418         if *f != nil {
 419                 lidMap = (*f).LID
 420         }
 421
 422         uid := <-uidChan
 423         if uid != 0 {
 424                 lidMap[uid] = lid
 425         }
 426
 427         *f = &File{
 428                 FID: fid,
 429
 430                 AID: aid,
 431                 EID: eid,
 432                 GID: gid,
 433                 LID: lidMap,
 434
 435                 EpisodeString: epstr,
 436                 EpisodeNumber: epno,
 437
 438                 RelatedEpisodes: related,
 439                 Deprecated:      ints[6] != 0,
 440
 441                 CRCMatch:   ints[7]&stateCRCOK != 0,
 442                 BadCRC:     ints[7]&stateCRCERR != 0,
 443                 Version:    version,
 444                 Uncensored: ints[7]&stateUncensored != 0,
 445                 Censored:   ints[7]&stateCensored != 0,
 446
 447                 Incomplete: video.Resolution.Empty(),
 448
 449                 Filesize: ints[8],
 450                 Ed2kHash: parts[9],
 451                 SHA1Hash: parts[10],
 452                 CRC32:    parts[11],
 453
 454                 Source: FileSource(parts[13]),
 455
 456                 AudioStreams:      streams,
 457                 SubtitleLanguages: slangs,
 458                 VideoInfo:         video,
 459                 FileExtension:     parts[19],
 460
 461                 Length:  time.Duration(ints[22]) * time.Second,
 462                 AirDate: time.Unix(ints[23], 0),
 463         }
 464         return true
 465 }