app/tasks/importtasks.py

   1 import flask, json
   2 from flask.ext.sqlalchemy import SQLAlchemy
   3 import urllib.request
   4 from urllib.parse import urlparse, quote_plus
   5 from app import app
   6 from app.models import *
   7 from app.tasks import celery
   8
   9 class TaskError(Exception):
  10         def __init__(self, value):
  11                 self.value = value
  12         def __str__(self):
  13                 return repr(self.value)
  14
  15
  16 class GithubURLMaker:
  17         def __init__(self, url):
  18                 # Rewrite path
  19                 import re
  20                 m = re.search("^\/([^\/]+)\/([^\/]+)\/?$", url.path)
  21                 if m is None:
  22                         return
  23
  24                 user = m.group(1)
  25                 repo = m.group(2)
  26                 self.baseUrl = "https://raw.githubusercontent.com/" + user + "/" + repo.replace(".git", "") + "/master"
  27                 self.user = user
  28                 self.repo = repo
  29
  30         def isValid(self):
  31                 return self.baseUrl is not None
  32
  33         def getRepoURL(self):
  34                 return "https://github.com/" + self.user + "/" + self.repo + ".git"
  35
  36         def getIssueTrackerURL(self):
  37                 return "https://github.com/" + self.user + "/" + self.repo + "/issues/"
  38
  39         def getModConfURL(self):
  40                 return self.baseUrl + "/mod.conf"
  41
  42         def getDescURL(self):
  43                 return self.baseUrl + "/description.txt"
  44
  45         def getScreenshotURL(self):
  46                 return self.baseUrl + "/screenshot.png"
  47
  48         def getCommitsURL(self, branch):
  49                 return "https://api.github.com/repos/" + self.user + "/" + self.repo + "/commits?sha" + urllib.parse.quote_plus(branch)
  50
  51         def getCommitDownload(self, commit):
  52                 return "https://github.com/" + self.user + "/" + self.repo + "/archive/" + commit + ".zip"
  53
  54
  55 krock_list_cache = None
  56 krock_list_cache_by_name = None
  57 def getKrockList():
  58         global krock_list_cache
  59         global krock_list_cache_by_name
  60
  61         if krock_list_cache is None:
  62                 contents = urllib.request.urlopen("http://krock-works.16mb.com/MTstuff/modList.php").read().decode("utf-8")
  63                 list = json.loads(contents)
  64
  65                 def h(x):
  66                         if not ("title"   in x and "author" in x and \
  67                                         "topicId" in x and "link"   in x and x["link"] != ""):
  68                                 return False
  69
  70                         import re
  71                         m = re.search("\[([A-Za-z0-9_]+)\]", x["title"])
  72                         if m is None:
  73                                 return False
  74
  75                         x["name"] = m.group(1)
  76                         return True
  77
  78                 def g(x):
  79                         return {
  80                                 "title":   x["title"],
  81                                 "author":  x["author"],
  82                                 "name":    x["name"],
  83                                 "topicId": x["topicId"],
  84                                 "link":    x["link"],
  85                         }
  86
  87                 krock_list_cache = [g(x) for x in list if h(x)]
  88                 krock_list_cache_by_name = {}
  89                 for x in krock_list_cache:
  90                         if not x["name"] in krock_list_cache_by_name:
  91                                 krock_list_cache_by_name[x["name"]] = []
  92
  93                         krock_list_cache_by_name[x["name"]].append(x)
  94
  95         return krock_list_cache, krock_list_cache_by_name
  96
  97 def findModInfo(author, name, link):
  98         _, lookup = getKrockList()
  99
 100         if name in lookup:
 101                 if len(lookup[name]) == 1:
 102                         return lookup[name][0]
 103
 104                 for x in lookup[name]:
 105                         if x["author"] == author:
 106                                 return x
 107
 108         return None
 109
 110
 111 def parseConf(string):
 112         retval = {}
 113         for line in string.split("\n"):
 114                 idx = line.find("=")
 115                 if idx > 0:
 116                         key   = line[:idx-1].strip()
 117                         value = line[idx+1:].strip()
 118                         retval[key] = value
 119
 120         return retval
 121
 122
 123 @celery.task()
 124 def getMeta(urlstr, author):
 125         url = urlparse(urlstr)
 126
 127         urlmaker = None
 128         if url.netloc == "github.com":
 129                 urlmaker = GithubURLMaker(url)
 130         else:
 131                 raise TaskError("Unsupported repo")
 132
 133         if not urlmaker.isValid():
 134                 raise TaskError("Error! Url maker not valid")
 135
 136         result = {}
 137
 138         result["repo"] = urlmaker.getRepoURL()
 139         result["issueTracker"] = urlmaker.getIssueTrackerURL()
 140
 141         try:
 142                 contents = urllib.request.urlopen(urlmaker.getModConfURL()).read().decode("utf-8")
 143                 conf = parseConf(contents)
 144                 for key in ["name", "description", "title"]:
 145                         try:
 146                                 result[key] = conf[key]
 147                         except KeyError:
 148                                 pass
 149         except OSError:
 150                 print("mod.conf does not exist")
 151
 152         if "name" in result:
 153                 result["title"] = result["name"].replace("_", " ").title()
 154
 155         if not "description" in result:
 156                 try:
 157                         contents = urllib.request.urlopen(urlmaker.getDescURL()).read().decode("utf-8")
 158                         result["description"] = contents.strip()
 159                 except OSError:
 160                         print("description.txt does not exist!")
 161
 162         if "description" in result:
 163                 desc = result["description"]
 164                 idx = desc.find(".") + 1
 165                 cutIdx = min(len(desc), 200 if idx < 5 else idx)
 166                 result["short_description"] = desc[:cutIdx]
 167
 168         info = findModInfo(author, result["name"], result["repo"])
 169         if info is not None:
 170                 result["forumId"] = info["topicId"]
 171
 172         return result
 173
 174 @celery.task()
 175 def makeVCSRelease(id, branch):
 176         release = PackageRelease.query.get(id)
 177         if release is None:
 178                 raise TaskError("No such release!")
 179
 180         if release.package is None:
 181                 raise TaskError("No package attached to release")
 182
 183         url = urlparse(release.package.repo)
 184
 185         urlmaker = None
 186         if url.netloc == "github.com":
 187                 urlmaker = GithubURLMaker(url)
 188         else:
 189                 raise TaskError("Unsupported repo")
 190
 191         if not urlmaker.isValid():
 192                 raise TaskError("Invalid github repo URL")
 193
 194         contents = urllib.request.urlopen(urlmaker.getCommitsURL(branch)).read().decode("utf-8")
 195         commits = json.loads(contents)
 196
 197         if len(commits) == 0:
 198                 raise TaskError("No commits found")
 199
 200         release.url = urlmaker.getCommitDownload(commits[0]["sha"])
 201         release.task_id = None
 202         db.session.commit()
 203
 204         return release.url