app/tasks/importtasks.py

   1 import flask, json
   2 from flask.ext.sqlalchemy import SQLAlchemy
   3 import urllib.request
   4 from urllib.parse import urlparse, quote_plus
   5 from app import app
   6 from app.models import *
   7 from app.tasks import celery
   8
   9 class TaskError(Exception):
  10         def __init__(self, value):
  11                 self.value = value
  12         def __str__(self):
  13                 return repr(self.value)
  14
  15
  16 class GithubURLMaker:
  17         def __init__(self, url):
  18                 # Rewrite path
  19                 import re
  20                 m = re.search("^\/([^\/]+)\/([^\/]+)\/?$", url.path)
  21                 if m is None:
  22                         return
  23
  24                 user = m.group(1)
  25                 repo = m.group(2)
  26                 self.baseUrl = "https://raw.githubusercontent.com/{}/{}/master" \
  27                                 .format(user, repo.replace(".git", ""))
  28                 self.user = user
  29                 self.repo = repo
  30
  31         def isValid(self):
  32                 return self.baseUrl is not None
  33
  34         def getRepoURL(self):
  35                 return "https://github.com/{}/{}".format(self.user, self.repo)
  36
  37         def getIssueTrackerURL(self):
  38                 return "https://github.com/{}/{}/issues/".format(self.user, self.repo)
  39
  40         def getModConfURL(self):
  41                 return self.baseUrl + "/mod.conf"
  42
  43         def getDescURL(self):
  44                 return self.baseUrl + "/description.txt"
  45
  46         def getScreenshotURL(self):
  47                 return self.baseUrl + "/screenshot.png"
  48
  49         def getCommitsURL(self, branch):
  50                 return "https://api.github.com/repos/{}/{}/commits?sha={}" \
  51                                 .format(self.user, self.repo, urllib.parse.quote_plus(branch))
  52
  53         def getCommitDownload(self, commit):
  54                 return "https://github.com/{}/{}/archive/{}.zip" \
  55                                 .format(self.user, self.repo, commit)
  56
  57
  58 krock_list_cache = None
  59 krock_list_cache_by_name = None
  60 def getKrockList():
  61         global krock_list_cache
  62         global krock_list_cache_by_name
  63
  64         if krock_list_cache is None:
  65                 contents = urllib.request.urlopen("http://krock-works.16mb.com/MTstuff/modList.php").read().decode("utf-8")
  66                 list = json.loads(contents)
  67
  68                 def h(x):
  69                         if not ("title"   in x and "author" in x and \
  70                                         "topicId" in x and "link"   in x and x["link"] != ""):
  71                                 return False
  72
  73                         import re
  74                         m = re.search("\[([A-Za-z0-9_]+)\]", x["title"])
  75                         if m is None:
  76                                 return False
  77
  78                         x["name"] = m.group(1)
  79                         return True
  80
  81                 def g(x):
  82                         return {
  83                                 "title":   x["title"],
  84                                 "author":  x["author"],
  85                                 "name":    x["name"],
  86                                 "topicId": x["topicId"],
  87                                 "link":    x["link"],
  88                         }
  89
  90                 krock_list_cache = [g(x) for x in list if h(x)]
  91                 krock_list_cache_by_name = {}
  92                 for x in krock_list_cache:
  93                         if not x["name"] in krock_list_cache_by_name:
  94                                 krock_list_cache_by_name[x["name"]] = []
  95
  96                         krock_list_cache_by_name[x["name"]].append(x)
  97
  98         return krock_list_cache, krock_list_cache_by_name
  99
 100 def findModInfo(author, name, link):
 101         _, lookup = getKrockList()
 102
 103         if name in lookup:
 104                 if len(lookup[name]) == 1:
 105                         return lookup[name][0]
 106
 107                 for x in lookup[name]:
 108                         if x["author"] == author:
 109                                 return x
 110
 111         return None
 112
 113
 114 def parseConf(string):
 115         retval = {}
 116         for line in string.split("\n"):
 117                 idx = line.find("=")
 118                 if idx > 0:
 119                         key   = line[:idx-1].strip()
 120                         value = line[idx+1:].strip()
 121                         retval[key] = value
 122
 123         return retval
 124
 125
 126 @celery.task()
 127 def getMeta(urlstr, author):
 128         url = urlparse(urlstr)
 129
 130         urlmaker = None
 131         if url.netloc == "github.com":
 132                 urlmaker = GithubURLMaker(url)
 133         else:
 134                 raise TaskError("Unsupported repo")
 135
 136         if not urlmaker.isValid():
 137                 raise TaskError("Error! Url maker not valid")
 138
 139         result = {}
 140
 141         result["repo"] = urlmaker.getRepoURL()
 142         result["issueTracker"] = urlmaker.getIssueTrackerURL()
 143
 144         try:
 145                 contents = urllib.request.urlopen(urlmaker.getModConfURL()).read().decode("utf-8")
 146                 conf = parseConf(contents)
 147                 for key in ["name", "description", "title"]:
 148                         try:
 149                                 result[key] = conf[key]
 150                         except KeyError:
 151                                 pass
 152         except OSError:
 153                 print("mod.conf does not exist")
 154
 155         if "name" in result:
 156                 result["title"] = result["name"].replace("_", " ").title()
 157
 158         if not "description" in result:
 159                 try:
 160                         contents = urllib.request.urlopen(urlmaker.getDescURL()).read().decode("utf-8")
 161                         result["description"] = contents.strip()
 162                 except OSError:
 163                         print("description.txt does not exist!")
 164
 165         if "description" in result:
 166                 desc = result["description"]
 167                 idx = desc.find(".") + 1
 168                 cutIdx = min(len(desc), 200 if idx < 5 else idx)
 169                 result["short_description"] = desc[:cutIdx]
 170
 171         info = findModInfo(author, result["name"], result["repo"])
 172         if info is not None:
 173                 result["forumId"] = info["topicId"]
 174
 175         return result
 176
 177 @celery.task()
 178 def makeVCSRelease(id, branch):
 179         release = PackageRelease.query.get(id)
 180         if release is None:
 181                 raise TaskError("No such release!")
 182
 183         if release.package is None:
 184                 raise TaskError("No package attached to release")
 185
 186         url = urlparse(release.package.repo)
 187
 188         urlmaker = None
 189         if url.netloc == "github.com":
 190                 urlmaker = GithubURLMaker(url)
 191         else:
 192                 raise TaskError("Unsupported repo")
 193
 194         if not urlmaker.isValid():
 195                 raise TaskError("Invalid github repo URL")
 196
 197         contents = urllib.request.urlopen(urlmaker.getCommitsURL(branch)).read().decode("utf-8")
 198         commits = json.loads(contents)
 199
 200         if len(commits) == 0:
 201                 raise TaskError("No commits found")
 202
 203         release.url = urlmaker.getCommitDownload(commits[0]["sha"])
 204         release.task_id = None
 205         db.session.commit()
 206
 207         return release.url