filecache.go

   1 package anidb
   2
   3 import (
   4         "fmt"
   5         "github.com/Kovensky/go-anidb/misc"
   6         "github.com/Kovensky/go-anidb/udp"
   7         "github.com/Kovensky/go-fscache"
   8         "image"
   9         "log"
  10         "regexp"
  11         "sort"
  12         "strconv"
  13         "strings"
  14         "time"
  15 )
  16
  17 var _ cacheable = &File{}
  18
  19 func (f *File) setCachedTS(ts time.Time) {
  20         f.Cached = ts
  21 }
  22
  23 func (f *File) IsStale() bool {
  24         if f == nil {
  25                 return true
  26         }
  27         if f.Incomplete {
  28                 return time.Now().Sub(f.Cached) > FileIncompleteCacheDuration
  29         }
  30         return time.Now().Sub(f.Cached) > FileCacheDuration
  31 }
  32
  33 func cacheFile(f *File) {
  34         CacheSet(f.FID, "fid", "by-ed2k", f.Ed2kHash, f.Filesize)
  35         CacheSet(f, "fid", f.FID)
  36 }
  37
  38 type FID int
  39
  40 func (fid FID) File() *File {
  41         var f File
  42         if CacheGet(&f, "fid", fid) == nil {
  43                 return &f
  44         }
  45         return nil
  46 }
  47
  48 // Prefetches the Anime, Episode and Group that this
  49 // file is linked to using the given AniDB instance.
  50 //
  51 // Returns a channel where this file will be sent to
  52 // when the prefetching is done; if the file is nil,
  53 // the channel will return nil.
  54 func (f *File) Prefetch(adb *AniDB) <-chan *File {
  55         ch := make(chan *File, 1)
  56         go func() {
  57                 if f != nil {
  58                         a := adb.AnimeByID(f.AID)
  59                         g := adb.GroupByID(f.GID)
  60                         <-a
  61                         <-g
  62                         ch <- f
  63                 }
  64                 close(ch)
  65         }()
  66         return ch
  67 }
  68
  69 // Retrieves a File by its FID. Uses the UDP API.
  70 func (adb *AniDB) FileByID(fid FID) <-chan *File {
  71         key := []fscache.CacheKey{"fid", fid}
  72
  73         ch := make(chan *File, 1)
  74
  75         if fid < 1 {
  76                 ch <- nil
  77                 close(ch)
  78                 return ch
  79         }
  80
  81         ic := make(chan notification, 1)
  82         go func() { ch <- (<-ic).(*File); close(ch) }()
  83         if intentMap.Intent(ic, key...) {
  84                 return ch
  85         }
  86
  87         if !Cache.IsValid(InvalidKeyCacheDuration, key...) {
  88                 intentMap.NotifyClose((*File)(nil), key...)
  89                 return ch
  90         }
  91
  92         f := fid.File()
  93         if !f.IsStale() {
  94                 intentMap.NotifyClose(f, key...)
  95                 return ch
  96         }
  97
  98         go func() {
  99                 reply := <-adb.udp.SendRecv("FILE",
 100                         paramMap{
 101                                 "fid":   fid,
 102                                 "fmask": fileFmask,
 103                                 "amask": fileAmask,
 104                         })
 105
 106                 if reply.Error() == nil {
 107                         f = adb.parseFileResponse(reply, false)
 108
 109                         cacheFile(f)
 110                 } else if reply.Code() == 320 {
 111                         Cache.SetInvalid(key...)
 112                 }
 113
 114                 intentMap.NotifyClose(f, key...)
 115         }()
 116         return ch
 117 }
 118
 119 var validEd2kHash = regexp.MustCompile(`\A[:xdigit:]{32}\z`)
 120
 121 // Retrieves a File by its Ed2kHash + Filesize combination. Uses the UDP API.
 122 func (adb *AniDB) FileByEd2kSize(ed2k string, size int64) <-chan *File {
 123         key := []fscache.CacheKey{"fid", "by-ed2k", ed2k, size}
 124
 125         ch := make(chan *File, 1)
 126
 127         if size < 1 || !validEd2kHash.MatchString(ed2k) {
 128                 ch <- nil
 129                 close(ch)
 130                 return ch
 131         }
 132         // AniDB always uses lower case hashes
 133         ed2k = strings.ToLower(ed2k)
 134
 135         ic := make(chan notification, 1)
 136         go func() {
 137                 fid := (<-ic).(FID)
 138                 if fid > 0 {
 139                         ch <- <-adb.FileByID(fid)
 140                 }
 141                 close(ch)
 142         }()
 143         if intentMap.Intent(ic, key...) {
 144                 return ch
 145         }
 146
 147         if !Cache.IsValid(InvalidKeyCacheDuration, key...) {
 148                 intentMap.NotifyClose(FID(0), key...)
 149                 return ch
 150         }
 151
 152         fid := FID(0)
 153
 154         switch ts, err := Cache.Get(&fid, key...); {
 155         case err != nil && time.Now().Sub(ts) < FileCacheDuration:
 156                 intentMap.NotifyClose(fid, key...)
 157                 return ch
 158         }
 159
 160         go func() {
 161                 reply := <-adb.udp.SendRecv("FILE",
 162                         paramMap{
 163                                 "ed2k":  ed2k,
 164                                 "size":  size,
 165                                 "fmask": fileFmask,
 166                                 "amask": fileAmask,
 167                         })
 168
 169                 var f *File
 170                 if reply.Error() == nil {
 171                         f = adb.parseFileResponse(reply, false)
 172
 173                         fid = f.FID
 174
 175                         cacheFile(f)
 176                 } else if reply.Code() == 320 { // file not found
 177                         Cache.SetInvalid(key...)
 178                 } else if reply.Code() == 322 { // multiple files found
 179                         panic("Don't know what to do with " + strings.Join(reply.Lines(), "\n"))
 180                 }
 181
 182                 intentMap.NotifyClose(fid, key...)
 183         }()
 184         return ch
 185 }
 186
 187 var fileFmask = "77da7fe8"
 188 var fileAmask = "00008000"
 189
 190 const (
 191         fileStateCRCOK = 1 << iota
 192         fileStateCRCERR
 193         fileStateV2
 194         fileStateV3
 195         fileStateV4
 196         fileStateV5
 197         fileStateUncensored
 198         fileStateCensored
 199 )
 200
 201 func sanitizeCodec(codec string) string {
 202         switch codec {
 203         case "MP3 CBR":
 204                 return "MP3"
 205         case "WMV9 (also WMV3)":
 206                 return "WMV9"
 207         case "Ogg (Vorbis)":
 208                 return "Vorbis"
 209         case "H264/AVC":
 210                 return "H.264"
 211         }
 212         return codec
 213 }
 214
 215 func (adb *AniDB) parseFileResponse(reply udpapi.APIReply, calledFromFIDsByGID bool) *File {
 216         if reply.Error() != nil {
 217                 return nil
 218         }
 219         if reply.Truncated() {
 220                 panic("Truncated")
 221         }
 222
 223         parts := strings.Split(reply.Lines()[1], "|")
 224         ints := make([]int64, len(parts))
 225         for i, p := range parts {
 226                 ints[i], _ = strconv.ParseInt(p, 10, 64)
 227         }
 228
 229         partial := false
 230
 231         rels := strings.Split(parts[4], "'")
 232         relList := make([]EID, 0, len(parts[4]))
 233         related := make(RelatedEpisodes, len(parts[4]))
 234         for _, rel := range rels {
 235                 r := strings.Split(rel, ",")
 236                 if len(r) < 2 {
 237                         continue
 238                 }
 239
 240                 eid, _ := strconv.ParseInt(r[0], 10, 32)
 241                 pct, _ := strconv.ParseInt(r[1], 10, 32)
 242                 relList = append(relList, EID(eid))
 243                 related[EID(eid)] = float32(pct) / 100
 244
 245                 if pct != 100 {
 246                         partial = true
 247                 }
 248         }
 249
 250         epno := misc.ParseEpisodeList(parts[23])
 251         fid := FID(ints[0])
 252         aid := AID(ints[1])
 253         eid := EID(ints[2])
 254         gid := GID(ints[3])
 255
 256         if !epno[0].Start.ContainsEpisodes(epno[0].End) || len(epno) > 1 || len(relList) > 0 {
 257                 // epno is broken -- we need to sanitize it
 258                 thisEp := <-adb.EpisodeByID(eid)
 259                 bad := false
 260                 if thisEp != nil {
 261                         parts := make([]string, 1, len(relList)+1)
 262                         parts[0] = thisEp.Episode.String()
 263
 264                         // everything after this SHOULD be cache hits now, unless this is somehow
 265                         // linked with an EID from a different anime (*stares at Haruhi*).
 266                         // We don't want to use eps from different AIDs anyway, so that makes
 267                         // the job easier.
 268
 269                         // We check if the related episodes are all in sequence from this one.
 270                         // If they are, we build a new epno with the sequence. Otherwise,
 271                         // our epno will only have the primary episode.
 272
 273                         // gather the episode numbers
 274                         for _, eid := range relList {
 275                                 if ep := eid.Episode(); ep != nil && ep.AID == thisEp.AID {
 276                                         parts = append(parts, ep.Episode.String())
 277                                 } else {
 278                                         bad = true
 279                                         break
 280                                 }
 281                         }
 282
 283                         test := misc.EpisodeList{}
 284                         // only if we didn't break the loop
 285                         if !bad {
 286                                 test = misc.ParseEpisodeList(strings.Join(parts, ","))
 287                         }
 288
 289                         if partial {
 290                                 if calledFromFIDsByGID {
 291                                         epno = test
 292                                         log.Printf("UDP!!! FID %d is only part of episode %s with no complementary files", fid, epno)
 293                                 } else if len(test) == 1 && test[0].Start.Number == test[0].End.Number {
 294                                         fids := []int{}
 295
 296                                         for fid := range adb.FIDsByGID(thisEp, gid) {
 297                                                 fids = append(fids, int(fid))
 298                                         }
 299                                         if len(fids) >= 1 && fids[0] == 0 {
 300                                                 fids = fids[1:]
 301                                                 // Only entry was API error
 302                                                 if len(fids) == 0 {
 303                                                         return nil
 304                                                 }
 305                                         }
 306                                         sort.Sort(sort.IntSlice(fids))
 307                                         idx := sort.SearchInts(fids, int(fid))
 308                                         if idx == len(fids) {
 309                                                 panic(fmt.Sprintf("FID %d couldn't locate itself", fid))
 310                                         }
 311
 312                                         epno = test
 313
 314                                         // equate pointers
 315                                         epno[0].End = epno[0].Start
 316
 317                                         epno[0].Start.Parts = len(fids)
 318                                         epno[0].Start.Part = idx
 319                                 } else {
 320                                         panic(fmt.Sprintf("Don't know what to do with partial episode %s (EID %d)", test, eid))
 321                                 }
 322                         } else {
 323                                 // if they're all in sequence, then we'll only have a single range in the list
 324                                 if len(test) == 1 {
 325                                         epno = test
 326                                 } else {
 327                                         // use only the primary epno then
 328                                         epno = misc.ParseEpisodeList(thisEp.Episode.String())
 329                                 }
 330                         }
 331                 }
 332         }
 333
 334         version := FileVersion(1)
 335         switch i := ints[6]; {
 336         case i&fileStateV5 != 0:
 337                 version = 5
 338         case i&fileStateV4 != 0:
 339                 version = 4
 340         case i&fileStateV3 != 0:
 341                 version = 3
 342         case i&fileStateV2 != 0:
 343                 version = 2
 344         }
 345
 346         // codecs (parts[13]), bitrates (ints[14]), langs (parts[19])
 347         codecs := strings.Split(parts[13], "'")
 348         bitrates := strings.Split(parts[14], "'")
 349         alangs := strings.Split(parts[19], "'")
 350         streams := make([]AudioStream, len(codecs))
 351         for i := range streams {
 352                 br, _ := strconv.ParseInt(bitrates[i], 10, 32)
 353                 streams[i] = AudioStream{
 354                         Bitrate:  int(br),
 355                         Codec:    sanitizeCodec(codecs[i]),
 356                         Language: Language(alangs[i]),
 357                 }
 358         }
 359
 360         sl := strings.Split(parts[20], "'")
 361         slangs := make([]Language, len(sl))
 362         for i := range sl {
 363                 slangs[i] = Language(sl[i])
 364         }
 365
 366         depth := int(ints[11])
 367         if depth == 0 {
 368                 depth = 8
 369         }
 370         res := strings.Split(parts[17], "x")
 371         width, _ := strconv.ParseInt(res[0], 10, 32)
 372         height, _ := strconv.ParseInt(res[1], 10, 32)
 373         video := VideoInfo{
 374                 Bitrate:    int(ints[16]),
 375                 Codec:      sanitizeCodec(parts[15]),
 376                 ColorDepth: depth,
 377                 Resolution: image.Rect(0, 0, int(width), int(height)),
 378         }
 379
 380         return &File{
 381                 FID: fid,
 382
 383                 AID: aid,
 384                 EID: eid,
 385                 GID: gid,
 386
 387                 EpisodeNumber: epno,
 388
 389                 RelatedEpisodes: related,
 390                 Deprecated:      ints[5] != 0,
 391
 392                 CRCMatch:   ints[6]&fileStateCRCOK != 0,
 393                 BadCRC:     ints[6]&fileStateCRCERR != 0,
 394                 Version:    version,
 395                 Uncensored: ints[6]&fileStateUncensored != 0,
 396                 Censored:   ints[6]&fileStateCensored != 0,
 397
 398                 Incomplete: video.Resolution.Empty(),
 399
 400                 Filesize: ints[7],
 401                 Ed2kHash: parts[8],
 402                 SHA1Hash: parts[9],
 403                 CRC32:    parts[10],
 404
 405                 Source: FileSource(parts[12]),
 406
 407                 AudioStreams:      streams,
 408                 SubtitleLanguages: slangs,
 409                 VideoInfo:         video,
 410                 FileExtension:     parts[18],
 411
 412                 Length:  time.Duration(ints[21]) * time.Second,
 413                 AirDate: time.Unix(ints[22], 0),
 414         }
 415 }