filecache.go

   1 package anidb
   2
   3 import (
   4         "fmt"
   5         "github.com/EliasFleckenstein03/go-anidb/misc"
   6         "github.com/EliasFleckenstein03/go-anidb/udp"
   7         "github.com/EliasFleckenstein03/go-fscache"
   8         "image"
   9         "regexp"
  10         "sort"
  11         "strconv"
  12         "strings"
  13         "time"
  14 )
  15
  16 var _ cacheable = &File{}
  17
  18 func (f *File) setCachedTS(ts time.Time) {
  19         f.Cached = ts
  20 }
  21
  22 func (f *File) IsStale() bool {
  23         if f == nil {
  24                 return true
  25         }
  26         if f.Incomplete {
  27                 return time.Now().Sub(f.Cached) > FileIncompleteCacheDuration
  28         }
  29         return time.Now().Sub(f.Cached) > FileCacheDuration
  30 }
  31
  32 func cacheFile(f *File) {
  33         CacheSet(f.AID, "aid", "by-eid", f.EID)
  34         CacheSet(f.FID, "fid", "by-ed2k", f.Ed2kHash, f.Filesize)
  35         CacheSet(f, "fid", f.FID)
  36 }
  37
  38 type FID int
  39
  40 func (fid FID) File() *File {
  41         var f File
  42         if CacheGet(&f, "fid", fid) == nil {
  43                 return &f
  44         }
  45         return nil
  46 }
  47
  48 // Prefetches the Anime, Episode and Group that this
  49 // file is linked to using the given AniDB instance.
  50 //
  51 // Returns a channel where this file will be sent to
  52 // when the prefetching is done; if the file is nil,
  53 // the channel will return nil.
  54 func (f *File) Prefetch(adb *AniDB) <-chan *File {
  55         ch := make(chan *File, 1)
  56         go func() {
  57                 if f != nil {
  58                         a := adb.AnimeByID(f.AID)
  59                         g := adb.GroupByID(f.GID)
  60                         <-a
  61                         <-g
  62                         ch <- f
  63                 }
  64                 close(ch)
  65         }()
  66         return ch
  67 }
  68
  69 // Retrieves a File by its FID. Uses the UDP API.
  70 func (adb *AniDB) FileByID(fid FID) <-chan *File {
  71         key := []fscache.CacheKey{"fid", fid}
  72
  73         ch := make(chan *File, 1)
  74
  75         if fid < 1 {
  76                 ch <- nil
  77                 close(ch)
  78                 return ch
  79         }
  80
  81         ic := make(chan notification, 1)
  82         go func() { ch <- (<-ic).(*File); close(ch) }()
  83         if intentMap.Intent(ic, key...) {
  84                 return ch
  85         }
  86
  87         if !Cache.IsValid(InvalidKeyCacheDuration, key...) {
  88                 intentMap.NotifyClose((*File)(nil), key...)
  89                 return ch
  90         }
  91
  92         f := fid.File()
  93         if !f.IsStale() {
  94                 intentMap.NotifyClose(f, key...)
  95                 return ch
  96         }
  97
  98         go func() {
  99                 reply := <-adb.udp.SendRecv("FILE",
 100                         paramMap{
 101                                 "fid":   fid,
 102                                 "fmask": fileFmask,
 103                                 "amask": fileAmask,
 104                         })
 105
 106                 if reply.Error() == nil {
 107                         adb.parseFileResponse(&f, reply, false)
 108
 109                         cacheFile(f)
 110                 } else if reply.Code() == 320 {
 111                         Cache.SetInvalid(key...)
 112                 }
 113
 114                 intentMap.NotifyClose(f, key...)
 115         }()
 116         return ch
 117 }
 118
 119 var validEd2kHash = regexp.MustCompile(`\A[[:xdigit:]]{32}\z`)
 120
 121 // Retrieves a File by its Ed2kHash + Filesize combination. Uses the UDP API.
 122 func (adb *AniDB) FileByEd2kSize(ed2k string, size int64) <-chan *File {
 123         key := []fscache.CacheKey{"fid", "by-ed2k", ed2k, size}
 124
 125         ch := make(chan *File, 1)
 126
 127         if size < 1 || !validEd2kHash.MatchString(ed2k) {
 128                 ch <- nil
 129                 close(ch)
 130                 return ch
 131         }
 132         // AniDB always uses lower case hashes
 133         ed2k = strings.ToLower(ed2k)
 134
 135         ic := make(chan notification, 1)
 136         go func() {
 137                 fid := (<-ic).(FID)
 138                 if fid > 0 {
 139                         ch <- <-adb.FileByID(fid)
 140                 }
 141                 close(ch)
 142         }()
 143         if intentMap.Intent(ic, key...) {
 144                 return ch
 145         }
 146
 147         if !Cache.IsValid(InvalidKeyCacheDuration, key...) {
 148                 intentMap.NotifyClose(FID(0), key...)
 149                 return ch
 150         }
 151
 152         fid := FID(0)
 153
 154         switch ts, err := Cache.Get(&fid, key...); {
 155         case err == nil && time.Now().Sub(ts) < FileCacheDuration:
 156                 intentMap.NotifyClose(fid, key...)
 157                 return ch
 158         }
 159
 160         go func() {
 161                 reply := <-adb.udp.SendRecv("FILE",
 162                         paramMap{
 163                                 "ed2k":  ed2k,
 164                                 "size":  size,
 165                                 "fmask": fileFmask,
 166                                 "amask": fileAmask,
 167                         })
 168
 169                 var f *File
 170                 if reply.Error() == nil {
 171                         adb.parseFileResponse(&f, reply, false)
 172
 173                         fid = f.FID
 174
 175                         cacheFile(f)
 176                 } else if reply.Code() == 320 { // file not found
 177                         Cache.SetInvalid(key...)
 178                 } else if reply.Code() == 322 { // multiple files found
 179                         panic("Don't know what to do with " + strings.Join(reply.Lines(), "\n"))
 180                 }
 181
 182                 intentMap.NotifyClose(fid, key...)
 183         }()
 184         return ch
 185 }
 186
 187 var fileFmask = "7fda7fe8"
 188 var fileAmask = "00008000"
 189
 190 const (
 191         stateCRCOK = 1 << iota
 192         stateCRCERR
 193         stateV2
 194         stateV3
 195         stateV4
 196         stateV5
 197         stateUncensored
 198         stateCensored
 199 )
 200
 201 func sanitizeCodec(codec string) string {
 202         switch codec {
 203         case "MP3 CBR":
 204                 return "MP3"
 205         case "WMV9 (also WMV3)":
 206                 return "WMV9"
 207         case "Ogg (Vorbis)":
 208                 return "Vorbis"
 209         case "H264/AVC":
 210                 return "H.264"
 211         }
 212         return codec
 213 }
 214
 215 var opedRE = regexp.MustCompile(`\A(Opening|Ending)(?: (\d+))?\z`)
 216
 217 func (adb *AniDB) parseFileResponse(f **File, reply udpapi.APIReply, calledFromFIDsByGID bool) bool {
 218         if reply.Error() != nil {
 219                 return false
 220         }
 221         if reply.Truncated() {
 222                 panic("Truncated")
 223         }
 224
 225         uidChan := make(chan UID, 1)
 226         if adb.udp.credentials != nil {
 227                 go func() { uidChan <- <-adb.GetUserUID(decrypt(adb.udp.credentials.username)) }()
 228         } else {
 229                 uidChan <- 0
 230                 close(uidChan)
 231         }
 232
 233         parts := strings.Split(reply.Lines()[1], "|")
 234         ints := make([]int64, len(parts))
 235         for i, p := range parts {
 236                 ints[i], _ = strconv.ParseInt(p, 10, 64)
 237         }
 238
 239         partial := false
 240
 241         rels := strings.Split(parts[5], "'")
 242         relList := make([]EID, 0, len(parts[5]))
 243         related := make(RelatedEpisodes, len(parts[5]))
 244         for _, rel := range rels {
 245                 r := strings.Split(rel, ",")
 246                 if len(r) < 2 {
 247                         continue
 248                 }
 249
 250                 eid, _ := strconv.ParseInt(r[0], 10, 32)
 251                 pct, _ := strconv.ParseInt(r[1], 10, 32)
 252                 relList = append(relList, EID(eid))
 253                 related[EID(eid)] = float32(pct) / 100
 254
 255                 if pct != 100 {
 256                         partial = true
 257                 }
 258         }
 259
 260         epno := misc.ParseEpisodeList(parts[24])
 261         fid := FID(ints[0])
 262         aid := AID(ints[1])
 263         eid := EID(ints[2])
 264         gid := GID(ints[3])
 265         lid := LID(ints[4])
 266
 267         if !epno[0].Start.ContainsEpisodes(epno[0].End) || len(epno) > 1 || len(relList) > 0 {
 268                 // epno is broken -- we need to sanitize it
 269                 thisEp := <-adb.EpisodeByID(eid)
 270                 bad := false
 271                 if thisEp != nil {
 272                         parts := make([]string, 1, len(relList)+1)
 273                         parts[0] = thisEp.Episode.String()
 274
 275                         // everything after this SHOULD be cache hits now, unless this is somehow
 276                         // linked with an EID from a different anime (*stares at Haruhi*).
 277                         // We don't want to use eps from different AIDs anyway, so that makes
 278                         // the job easier.
 279
 280                         // We check if the related episodes are all in sequence from this one.
 281                         // If they are, we build a new epno with the sequence. Otherwise,
 282                         // our epno will only have the primary episode.
 283
 284                         // gather the episode numbers
 285                         for _, eid := range relList {
 286                                 if ep := eid.Episode(); ep != nil && ep.AID == thisEp.AID {
 287                                         parts = append(parts, ep.Episode.String())
 288                                 } else {
 289                                         bad = true
 290                                         break
 291                                 }
 292                         }
 293
 294                         test := misc.EpisodeList{}
 295                         // only if we didn't break the loop
 296                         if !bad {
 297                                 test = misc.ParseEpisodeList(strings.Join(parts, ","))
 298                         }
 299
 300                         if partial {
 301                                 if calledFromFIDsByGID {
 302                                         epno = test
 303                                         adb.Logger.Printf("UDP!!! FID %d is only part of episode %s with no complementary files", fid, epno)
 304                                 } else if len(test) == 1 && test[0].Start.Number == test[0].End.Number {
 305                                         fids := []int{}
 306
 307                                         for fid := range adb.FIDsByGID(thisEp, gid) {
 308                                                 fids = append(fids, int(fid))
 309                                         }
 310                                         if len(fids) >= 1 && fids[0] == 0 {
 311                                                 fids = fids[1:]
 312                                                 // Only entry was API error
 313                                                 if len(fids) == 0 {
 314                                                         return false
 315                                                 }
 316                                         }
 317                                         sort.Sort(sort.IntSlice(fids))
 318                                         idx := sort.SearchInts(fids, int(fid))
 319                                         if idx == len(fids) {
 320                                                 panic(fmt.Sprintf("FID %d couldn't locate itself", fid))
 321                                         }
 322
 323                                         epno = test
 324
 325                                         // equate pointers
 326                                         epno[0].End = epno[0].Start
 327
 328                                         epno[0].Start.Parts = len(fids)
 329                                         epno[0].Start.Part = idx
 330                                 } else {
 331                                         panic(fmt.Sprintf("Don't know what to do with partial episode %s (EID %d)", test, eid))
 332                                 }
 333                         } else {
 334                                 // if they're all in sequence, then we'll only have a single range in the list
 335                                 if len(test) == 1 {
 336                                         epno = test
 337                                 } else {
 338                                         // use only the primary epno then
 339                                         epno = misc.ParseEpisodeList(thisEp.Episode.String())
 340                                 }
 341                         }
 342                 }
 343         }
 344
 345         epstr := epno.String()
 346         if len(epno) == 1 && epno[0].Type == misc.EpisodeTypeCredits && epno[0].Len() == 1 {
 347                 typ := ""
 348                 n := 0
 349
 350                 if ep := <-adb.EpisodeByID(eid); ep == nil {
 351                 } else if m := opedRE.FindStringSubmatch(ep.Titles["en"]); len(m) > 2 {
 352                         num, err := strconv.ParseInt(m[2], 10, 32)
 353                         if err == nil {
 354                                 n = int(num)
 355                         }
 356
 357                         typ = m[1]
 358                 }
 359
 360                 gobi := fmt.Sprintf("%d", n)
 361                 if n == 0 {
 362                         gobi = ""
 363                 }
 364
 365                 switch typ {
 366                 case "Opening":
 367                         epstr = "OP" + gobi
 368                 case "Ending":
 369                         epstr = "ED" + gobi
 370                 }
 371         }
 372
 373         version := FileVersion(1)
 374         switch i := ints[7]; {
 375         case i&stateV5 != 0:
 376                 version = 5
 377         case i&stateV4 != 0:
 378                 version = 4
 379         case i&stateV3 != 0:
 380                 version = 3
 381         case i&stateV2 != 0:
 382                 version = 2
 383         }
 384
 385         codecs := strings.Split(parts[14], "'")
 386         bitrates := strings.Split(parts[15], "'")
 387         alangs := strings.Split(parts[20], "'")
 388         streams := make([]AudioStream, len(codecs))
 389         for i := range streams {
 390                 br, _ := strconv.ParseInt(bitrates[i], 10, 32)
 391                 streams[i] = AudioStream{
 392                         Bitrate:  int(br),
 393                         Codec:    sanitizeCodec(codecs[i]),
 394                         Language: Language(alangs[i]),
 395                 }
 396         }
 397
 398         sl := strings.Split(parts[21], "'")
 399         slangs := make([]Language, len(sl))
 400         for i := range sl {
 401                 slangs[i] = Language(sl[i])
 402         }
 403
 404         depth := int(ints[12])
 405         if depth == 0 {
 406                 depth = 8
 407         }
 408         res := strings.Split(parts[18], "x")
 409         width, _ := strconv.ParseInt(res[0], 10, 32)
 410         height, _ := strconv.ParseInt(res[1], 10, 32)
 411         video := VideoInfo{
 412                 Bitrate:    int(ints[17]),
 413                 Codec:      sanitizeCodec(parts[16]),
 414                 ColorDepth: depth,
 415                 Resolution: image.Rect(0, 0, int(width), int(height)),
 416         }
 417
 418         lidMap := LIDMap{}
 419         if *f != nil {
 420                 lidMap = (*f).LID
 421         }
 422
 423         uid := <-uidChan
 424         if uid != 0 {
 425                 lidMap[uid] = lid
 426         }
 427
 428         *f = &File{
 429                 FID: fid,
 430
 431                 AID: aid,
 432                 EID: eid,
 433                 GID: gid,
 434                 LID: lidMap,
 435
 436                 EpisodeString: epstr,
 437                 EpisodeNumber: epno,
 438
 439                 RelatedEpisodes: related,
 440                 Deprecated:      ints[6] != 0,
 441
 442                 CRCMatch:   ints[7]&stateCRCOK != 0,
 443                 BadCRC:     ints[7]&stateCRCERR != 0,
 444                 Version:    version,
 445                 Uncensored: ints[7]&stateUncensored != 0,
 446                 Censored:   ints[7]&stateCensored != 0,
 447
 448                 Incomplete: video.Resolution.Empty(),
 449
 450                 Filesize: ints[8],
 451                 Ed2kHash: parts[9],
 452                 SHA1Hash: parts[10],
 453                 CRC32:    parts[11],
 454
 455                 Source: FileSource(parts[13]),
 456
 457                 AudioStreams:      streams,
 458                 SubtitleLanguages: slangs,
 459                 VideoInfo:         video,
 460                 FileExtension:     parts[19],
 461
 462                 Length:  time.Duration(ints[22]) * time.Second,
 463                 AirDate: time.Unix(ints[23], 0),
 464         }
 465         return true
 466 }