titles/titles.go

   1 package titles
   2
   3 import (
   4         "bufio"
   5         "bytes"
   6         "compress/gzip"
   7         "encoding/json"
   8         "io"
   9         "io/ioutil"
  10         "regexp"
  11         "sort"
  12         "strconv"
  13         "strings"
  14         "sync"
  15         "time"
  16 )
  17
  18 var _ = json.Decoder{}
  19
  20 const (
  21         DataDumpURL = "http://anidb.net/api/anime-titles.dat.gz"
  22 )
  23
  24 // Anime ID
  25 type AID int
  26
  27 type Name struct {
  28         Language string // ISO-ish language string
  29         Title    string
  30 }
  31
  32 type Anime struct {
  33         AID          AID
  34         PrimaryTitle string // The primary title ("x-jat main" title in the HTTP API)
  35
  36         OfficialNames map[string][]Name
  37         Synonyms      map[string][]Name
  38         ShortNames    map[string][]Name
  39 }
  40
  41 // Maps titles in the given language to AIDs
  42 type TitleMap struct {
  43         Language string // ISO-ish language string
  44
  45         OfficialMap map[string]AID
  46         SynonymMap  map[string]AID
  47         ShortMap    map[string]AID
  48 }
  49
  50 type TitlesDatabase struct {
  51         sync.RWMutex
  52         UpdateTime time.Time
  53         Languages  []string // List of all the languages present in the database
  54
  55         LanguageMap map[string]*TitleMap // Per-language map (key is ISO-ish language string)
  56         PrimaryMap  map[string]AID       // Primary title to AID map (language is always "x-jat")
  57
  58         AnimeMap map[AID]*Anime
  59 }
  60
  61 var createdRegexp = regexp.MustCompile(`^# created: (.*)$`)
  62
  63 // Loads the database from the given io.ReadCloser.
  64 //
  65 // The ReadCloser must point to a file or stream with
  66 // the contents of anime-titles.dat, which can be obtained
  67 // from the DataDumpURL. LoadDB will automatically try to
  68 // un-gzip, so the file can be stored in gzip format.
  69 //
  70 // Note: LoadDB will read the entire contents of the given
  71 // io.ReadCloser. LoadDB will call Close() on the
  72 // io.ReadCloser after reading it.
  73 func (db *TitlesDatabase) LoadDB(r io.ReadCloser) {
  74         db.Lock()
  75         defer db.Unlock()
  76
  77         all, _ := ioutil.ReadAll(r)
  78         r.Close()
  79
  80         var rd io.Reader
  81         if gz, err := gzip.NewReader(bytes.NewReader(all)); err == nil {
  82                 defer gz.Close()
  83                 rd = gz
  84         } else {
  85                 rd = bytes.NewReader(all)
  86         }
  87         sc := bufio.NewScanner(rd)
  88
  89         if db.PrimaryMap == nil {
  90                 db.PrimaryMap = map[string]AID{}
  91         }
  92         if db.LanguageMap == nil {
  93                 db.LanguageMap = map[string]*TitleMap{}
  94         }
  95         if db.AnimeMap == nil {
  96                 db.AnimeMap = map[AID]*Anime{}
  97         }
  98
  99         allLangs := map[string]struct{}{}
 100         for sc.Scan() {
 101                 s := sc.Text()
 102
 103                 if s[0] == '#' {
 104                         cr := createdRegexp.FindStringSubmatch(s)
 105
 106                         if len(cr) > 1 && cr[1] != "" {
 107                                 db.UpdateTime, _ = time.Parse(time.ANSIC, cr[1])
 108                         }
 109                         continue
 110                 }
 111
 112                 parts := strings.Split(s, "|")
 113                 if len(parts) < 4 {
 114                         continue
 115                 }
 116
 117                 aid, _ := strconv.ParseInt(parts[0], 10, 32)
 118                 typ, _ := strconv.ParseInt(parts[1], 10, 8)
 119
 120                 if _, ok := db.AnimeMap[AID(aid)]; !ok {
 121                         db.AnimeMap[AID(aid)] = &Anime{
 122                                 AID:           AID(aid),
 123                                 OfficialNames: map[string][]Name{},
 124                                 Synonyms:      map[string][]Name{},
 125                                 ShortNames:    map[string][]Name{},
 126                         }
 127                 }
 128
 129                 lang, title := parts[2], parts[3]
 130                 allLangs[lang] = struct{}{}
 131
 132                 switch typ {
 133                 case 1: // primary
 134                         db.PrimaryMap[title] = AID(aid)
 135
 136                         db.AnimeMap[AID(aid)].PrimaryTitle = strings.Replace(title, "`", "'", -1)
 137                 case 2: // synonym
 138                         lm, ok := db.LanguageMap[lang]
 139                         if !ok {
 140                                 lm = db.makeLangMap(lang)
 141                         }
 142                         lm.SynonymMap[title] = AID(aid)
 143
 144                         db.AnimeMap[AID(aid)].Synonyms[lang] = append(db.AnimeMap[AID(aid)].Synonyms[lang],
 145                                 Name{Language: lang, Title: strings.Replace(title, "`", "'", -1)})
 146                 case 3: // short
 147                         lm, ok := db.LanguageMap[lang]
 148                         if !ok {
 149                                 lm = db.makeLangMap(lang)
 150                         }
 151                         lm.ShortMap[title] = AID(aid)
 152
 153                         db.AnimeMap[AID(aid)].ShortNames[lang] = append(db.AnimeMap[AID(aid)].Synonyms[lang],
 154                                 Name{Language: lang, Title: strings.Replace(title, "`", "'", -1)})
 155                 case 4: // official
 156                         lm, ok := db.LanguageMap[lang]
 157                         if !ok {
 158                                 lm = db.makeLangMap(lang)
 159                         }
 160                         lm.OfficialMap[title] = AID(aid)
 161
 162                         db.AnimeMap[AID(aid)].OfficialNames[lang] = append(db.AnimeMap[AID(aid)].Synonyms[lang],
 163                                 Name{Language: lang, Title: strings.Replace(title, "`", "'", -1)})
 164                 }
 165         }
 166         langs := make([]string, 0, len(allLangs))
 167         for k, _ := range allLangs {
 168                 langs = append(langs, k)
 169         }
 170         sort.Strings(langs)
 171         db.Languages = langs
 172 }
 173
 174 func (db *TitlesDatabase) makeLangMap(lang string) *TitleMap {
 175         tm := &TitleMap{
 176                 Language:    lang,
 177                 OfficialMap: map[string]AID{},
 178                 SynonymMap:  map[string]AID{},
 179                 ShortMap:    map[string]AID{},
 180         }
 181         db.LanguageMap[lang] = tm
 182         return tm
 183 }