app/tasks/importtasks.py

   1 import flask, json, os
   2 from flask.ext.sqlalchemy import SQLAlchemy
   3 from urllib.error import HTTPError
   4 import urllib.request
   5 from urllib.parse import urlparse, quote_plus
   6 from app import app
   7 from app.models import *
   8 from app.tasks import celery, TaskError
   9 from app.utils import randomString
  10
  11 class GithubURLMaker:
  12         def __init__(self, url):
  13                 # Rewrite path
  14                 import re
  15                 m = re.search("^\/([^\/]+)\/([^\/]+)\/?$", url.path)
  16                 if m is None:
  17                         return
  18
  19                 user = m.group(1)
  20                 repo = m.group(2)
  21                 self.baseUrl = "https://raw.githubusercontent.com/{}/{}/master" \
  22                                 .format(user, repo.replace(".git", ""))
  23                 self.user = user
  24                 self.repo = repo
  25
  26         def isValid(self):
  27                 return self.baseUrl is not None
  28
  29         def getRepoURL(self):
  30                 return "https://github.com/{}/{}".format(self.user, self.repo)
  31
  32         def getIssueTrackerURL(self):
  33                 return "https://github.com/{}/{}/issues/".format(self.user, self.repo)
  34
  35         def getModConfURL(self):
  36                 return self.baseUrl + "/mod.conf"
  37
  38         def getDescURL(self):
  39                 return self.baseUrl + "/description.txt"
  40
  41         def getScreenshotURL(self):
  42                 return self.baseUrl + "/screenshot.png"
  43
  44         def getCommitsURL(self, branch):
  45                 return "https://api.github.com/repos/{}/{}/commits?sha={}" \
  46                                 .format(self.user, self.repo, urllib.parse.quote_plus(branch))
  47
  48         def getCommitDownload(self, commit):
  49                 return "https://github.com/{}/{}/archive/{}.zip" \
  50                                 .format(self.user, self.repo, commit)
  51
  52
  53 krock_list_cache = None
  54 krock_list_cache_by_name = None
  55 def getKrockList():
  56         global krock_list_cache
  57         global krock_list_cache_by_name
  58
  59         if krock_list_cache is None:
  60                 contents = urllib.request.urlopen("http://krock-works.16mb.com/MTstuff/modList.php").read().decode("utf-8")
  61                 list = json.loads(contents)
  62
  63                 def h(x):
  64                         if not ("title"   in x and "author" in x and \
  65                                         "topicId" in x and "link"   in x and x["link"] != ""):
  66                                 return False
  67
  68                         import re
  69                         m = re.search("\[([A-Za-z0-9_]+)\]", x["title"])
  70                         if m is None:
  71                                 return False
  72
  73                         x["name"] = m.group(1)
  74                         return True
  75
  76                 def g(x):
  77                         return {
  78                                 "title":   x["title"],
  79                                 "author":  x["author"],
  80                                 "name":    x["name"],
  81                                 "topicId": x["topicId"],
  82                                 "link":    x["link"],
  83                         }
  84
  85                 krock_list_cache = [g(x) for x in list if h(x)]
  86                 krock_list_cache_by_name = {}
  87                 for x in krock_list_cache:
  88                         if not x["name"] in krock_list_cache_by_name:
  89                                 krock_list_cache_by_name[x["name"]] = []
  90
  91                         krock_list_cache_by_name[x["name"]].append(x)
  92
  93         return krock_list_cache, krock_list_cache_by_name
  94
  95 def findModInfo(author, name, link):
  96         list, lookup = getKrockList()
  97
  98         if name is not None and name in lookup:
  99                 if len(lookup[name]) == 1:
 100                         return lookup[name][0]
 101
 102                 for x in lookup[name]:
 103                         if x["author"] == author:
 104                                 return x
 105
 106         if link is not None and len(link) > 15:
 107                 for x in list:
 108                         if link in x["link"]:
 109                                 return x
 110
 111         return None
 112
 113
 114 def parseConf(string):
 115         retval = {}
 116         for line in string.split("\n"):
 117                 idx = line.find("=")
 118                 if idx > 0:
 119                         key   = line[:idx].strip()
 120                         value = line[idx+1:].strip()
 121                         retval[key] = value
 122
 123         return retval
 124
 125
 126 @celery.task()
 127 def getMeta(urlstr, author):
 128         url = urlparse(urlstr)
 129
 130         urlmaker = None
 131         if url.netloc == "github.com":
 132                 urlmaker = GithubURLMaker(url)
 133         else:
 134                 raise TaskError("Unsupported repo")
 135
 136         if not urlmaker.isValid():
 137                 raise TaskError("Error! Url maker not valid")
 138
 139         result = {}
 140
 141         result["repo"] = urlmaker.getRepoURL()
 142         result["issueTracker"] = urlmaker.getIssueTrackerURL()
 143
 144         try:
 145                 contents = urllib.request.urlopen(urlmaker.getModConfURL()).read().decode("utf-8")
 146                 conf = parseConf(contents)
 147                 for key in ["name", "description", "title"]:
 148                         try:
 149                                 result[key] = conf[key]
 150                         except KeyError:
 151                                 pass
 152         except HTTPError:
 153                 print("mod.conf does not exist")
 154
 155         if "name" in result:
 156                 result["title"] = result["name"].replace("_", " ").title()
 157
 158         if not "description" in result:
 159                 try:
 160                         contents = urllib.request.urlopen(urlmaker.getDescURL()).read().decode("utf-8")
 161                         result["description"] = contents.strip()
 162                 except HTTPError:
 163                         print("description.txt does not exist!")
 164
 165         if "description" in result:
 166                 desc = result["description"]
 167                 idx = desc.find(".") + 1
 168                 cutIdx = min(len(desc), 200 if idx < 5 else idx)
 169                 result["short_description"] = desc[:cutIdx]
 170
 171         info = findModInfo(author, result.get("name"), result["repo"])
 172         if info is not None:
 173                 result["forumId"] = info.get("topicId")
 174
 175         return result
 176
 177
 178 @celery.task()
 179 def makeVCSRelease(id, branch):
 180         release = PackageRelease.query.get(id)
 181
 182         if release is None:
 183                 raise TaskError("No such release!")
 184
 185         if release.package is None:
 186                 raise TaskError("No package attached to release")
 187
 188         url = urlparse(release.package.repo)
 189
 190         urlmaker = None
 191         if url.netloc == "github.com":
 192                 urlmaker = GithubURLMaker(url)
 193         else:
 194                 raise TaskError("Unsupported repo")
 195
 196         if not urlmaker.isValid():
 197                 raise TaskError("Invalid github repo URL")
 198
 199         contents = urllib.request.urlopen(urlmaker.getCommitsURL(branch)).read().decode("utf-8")
 200         commits = json.loads(contents)
 201
 202         if len(commits) == 0:
 203                 raise TaskError("No commits found")
 204
 205         release.url = urlmaker.getCommitDownload(commits[0]["sha"])
 206         release.task_id = None
 207         db.session.commit()
 208
 209         return release.url
 210
 211
 212 @celery.task()
 213 def importRepoScreenshot(id):
 214         package = Package.query.get(id)
 215         if package is None:
 216                 raise Exception("Unexpected none package")
 217
 218         # Get URL Maker
 219         url = urlparse(package.repo)
 220         urlmaker = None
 221         if url.netloc == "github.com":
 222                 urlmaker = GithubURLMaker(url)
 223         else:
 224                 raise TaskError("Unsupported repo")
 225
 226         if not urlmaker.isValid():
 227                 raise TaskError("Error! Url maker not valid")
 228
 229         try:
 230                 filename = randomString(10) + ".png"
 231                 imagePath = os.path.join(app.config["UPLOAD_FOLDER"], filename)
 232                 print(imagePath)
 233                 urllib.request.urlretrieve(urlmaker.getScreenshotURL(), imagePath)
 234
 235                 ss = PackageScreenshot()
 236                 ss.package = package
 237                 ss.title   = "screenshot.png"
 238                 ss.url     = "/uploads/" + filename
 239                 db.session.add(ss)
 240                 db.session.commit()
 241
 242                 return "/uploads/" + filename
 243         except HTTPError:
 244                 print("screenshot.png does not exist")
 245
 246         return None