titles/titles.go

   1 // Implements parsing and searching of the anime-titles.dat database.
   2 //
   3 // http://wiki.anidb.info/w/API#Anime_Titles
   4 package titles
   5
   6 import (
   7         "bufio"
   8         "bytes"
   9         "compress/gzip"
  10         "encoding/json"
  11         "io"
  12         "io/ioutil"
  13         "regexp"
  14         "sort"
  15         "strconv"
  16         "strings"
  17         "sync"
  18         "time"
  19 )
  20
  21 var _ = json.Decoder{}
  22
  23 const (
  24         DataDumpURL = "http://anidb.net/api/anime-titles.dat.gz"
  25 )
  26
  27 // Anime ID
  28 type AID int
  29
  30 type Name struct {
  31         Language string // ISO-ish language string
  32         Title    string
  33 }
  34
  35 type Anime struct {
  36         AID          AID
  37         PrimaryTitle string // The primary title ("x-jat main" title in the HTTP API)
  38
  39         OfficialNames map[string][]Name
  40         Synonyms      map[string][]Name
  41         ShortNames    map[string][]Name
  42 }
  43
  44 // Maps titles in the given language to AIDs
  45 type TitleMap struct {
  46         Language string // ISO-ish language string
  47
  48         OfficialMap map[string]AID
  49         SynonymMap  map[string]AID
  50         ShortMap    map[string]AID
  51 }
  52
  53 type TitlesDatabase struct {
  54         sync.RWMutex
  55         UpdateTime time.Time
  56         Languages  []string // List of all the languages present in the database
  57
  58         LanguageMap map[string]*TitleMap // Per-language map (key is ISO-ish language string)
  59         PrimaryMap  map[string]AID       // Primary title to AID map (language is always "x-jat")
  60
  61         AnimeMap map[AID]*Anime
  62 }
  63
  64 var createdRegexp = regexp.MustCompile(`^# created: (.*)$`)
  65
  66 // Loads the database from the given io.Reader.
  67 //
  68 // The Reader must point to a file or stream with
  69 // the contents of anime-titles.dat, which can be obtained
  70 // from the DataDumpURL. LoadDB will automatically try to
  71 // un-gzip, so the file can be stored in gzip format.
  72 //
  73 // Note: LoadDB will read the entire contents of the given
  74 // io.Reader.
  75 func (db *TitlesDatabase) LoadDB(r io.Reader) {
  76         db.Lock()
  77         defer db.Unlock()
  78
  79         all, _ := ioutil.ReadAll(r)
  80
  81         var rd io.Reader
  82         if gz, err := gzip.NewReader(bytes.NewReader(all)); err == nil {
  83                 defer gz.Close()
  84                 rd = gz
  85         } else {
  86                 rd = bytes.NewReader(all)
  87         }
  88         sc := bufio.NewScanner(rd)
  89
  90         if db.PrimaryMap == nil {
  91                 db.PrimaryMap = map[string]AID{}
  92         }
  93         if db.LanguageMap == nil {
  94                 db.LanguageMap = map[string]*TitleMap{}
  95         }
  96         if db.AnimeMap == nil {
  97                 db.AnimeMap = map[AID]*Anime{}
  98         }
  99
 100         allLangs := map[string]struct{}{}
 101         for sc.Scan() {
 102                 s := sc.Text()
 103
 104                 if s[0] == '#' {
 105                         cr := createdRegexp.FindStringSubmatch(s)
 106
 107                         if len(cr) > 1 && cr[1] != "" {
 108                                 db.UpdateTime, _ = time.Parse(time.ANSIC, cr[1])
 109                         }
 110                         continue
 111                 }
 112
 113                 parts := strings.Split(s, "|")
 114                 if len(parts) < 4 {
 115                         continue
 116                 }
 117
 118                 aid, _ := strconv.ParseInt(parts[0], 10, 32)
 119                 typ, _ := strconv.ParseInt(parts[1], 10, 8)
 120
 121                 if _, ok := db.AnimeMap[AID(aid)]; !ok {
 122                         db.AnimeMap[AID(aid)] = &Anime{
 123                                 AID:           AID(aid),
 124                                 OfficialNames: map[string][]Name{},
 125                                 Synonyms:      map[string][]Name{},
 126                                 ShortNames:    map[string][]Name{},
 127                         }
 128                 }
 129
 130                 lang, title := parts[2], parts[3]
 131                 allLangs[lang] = struct{}{}
 132
 133                 switch typ {
 134                 case 1: // primary
 135                         db.PrimaryMap[title] = AID(aid)
 136
 137                         db.AnimeMap[AID(aid)].PrimaryTitle = strings.Replace(title, "`", "'", -1)
 138                 case 2: // synonym
 139                         lm, ok := db.LanguageMap[lang]
 140                         if !ok {
 141                                 lm = db.makeLangMap(lang)
 142                         }
 143                         lm.SynonymMap[title] = AID(aid)
 144
 145                         db.AnimeMap[AID(aid)].Synonyms[lang] = append(db.AnimeMap[AID(aid)].Synonyms[lang],
 146                                 Name{Language: lang, Title: strings.Replace(title, "`", "'", -1)})
 147                 case 3: // short
 148                         lm, ok := db.LanguageMap[lang]
 149                         if !ok {
 150                                 lm = db.makeLangMap(lang)
 151                         }
 152                         lm.ShortMap[title] = AID(aid)
 153
 154                         db.AnimeMap[AID(aid)].ShortNames[lang] = append(db.AnimeMap[AID(aid)].Synonyms[lang],
 155                                 Name{Language: lang, Title: strings.Replace(title, "`", "'", -1)})
 156                 case 4: // official
 157                         lm, ok := db.LanguageMap[lang]
 158                         if !ok {
 159                                 lm = db.makeLangMap(lang)
 160                         }
 161                         lm.OfficialMap[title] = AID(aid)
 162
 163                         db.AnimeMap[AID(aid)].OfficialNames[lang] = append(db.AnimeMap[AID(aid)].Synonyms[lang],
 164                                 Name{Language: lang, Title: strings.Replace(title, "`", "'", -1)})
 165                 }
 166         }
 167         langs := make([]string, 0, len(allLangs))
 168         for k, _ := range allLangs {
 169                 langs = append(langs, k)
 170         }
 171         sort.Strings(langs)
 172         db.Languages = langs
 173 }
 174
 175 func (db *TitlesDatabase) makeLangMap(lang string) *TitleMap {
 176         tm := &TitleMap{
 177                 Language:    lang,
 178                 OfficialMap: map[string]AID{},
 179                 SynonymMap:  map[string]AID{},
 180                 ShortMap:    map[string]AID{},
 181         }
 182         db.LanguageMap[lang] = tm
 183         return tm
 184 }