app/tasks/importtasks.py

   1 # Content DB
   2 # Copyright (C) 2018  rubenwardy
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU General Public License as published by
   6 # the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU General Public License
  15 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
  16
  17
  18 import flask, json, os
  19 from flask.ext.sqlalchemy import SQLAlchemy
  20 from urllib.error import HTTPError
  21 import urllib.request
  22 from urllib.parse import urlparse, quote_plus
  23 from app import app
  24 from app.models import *
  25 from app.tasks import celery, TaskError
  26 from app.utils import randomString
  27
  28 class GithubURLMaker:
  29         def __init__(self, url):
  30                 # Rewrite path
  31                 import re
  32                 m = re.search("^\/([^\/]+)\/([^\/]+)\/?$", url.path)
  33                 if m is None:
  34                         return
  35
  36                 user = m.group(1)
  37                 repo = m.group(2).replace(".git", "")
  38                 self.baseUrl = "https://raw.githubusercontent.com/{}/{}/master" \
  39                                 .format(user, repo)
  40                 self.user = user
  41                 self.repo = repo
  42
  43         def isValid(self):
  44                 return self.baseUrl is not None
  45
  46         def getRepoURL(self):
  47                 return "https://github.com/{}/{}".format(self.user, self.repo)
  48
  49         def getIssueTrackerURL(self):
  50                 return "https://github.com/{}/{}/issues/".format(self.user, self.repo)
  51
  52         def getModConfURL(self):
  53                 return self.baseUrl + "/mod.conf"
  54
  55         def getDescURL(self):
  56                 return self.baseUrl + "/description.txt"
  57
  58         def getScreenshotURL(self):
  59                 return self.baseUrl + "/screenshot.png"
  60
  61         def getCommitsURL(self, branch):
  62                 return "https://api.github.com/repos/{}/{}/commits?sha={}" \
  63                                 .format(self.user, self.repo, urllib.parse.quote_plus(branch))
  64
  65         def getCommitDownload(self, commit):
  66                 return "https://github.com/{}/{}/archive/{}.zip" \
  67                                 .format(self.user, self.repo, commit)
  68
  69
  70 krock_list_cache = None
  71 krock_list_cache_by_name = None
  72 def getKrockList():
  73         global krock_list_cache
  74         global krock_list_cache_by_name
  75
  76         if krock_list_cache is None:
  77                 contents = urllib.request.urlopen("http://krock-works.16mb.com/MTstuff/modList.php").read().decode("utf-8")
  78                 list = json.loads(contents)
  79
  80                 def h(x):
  81                         if not ("title"   in x and "author" in x and \
  82                                         "topicId" in x and "link"   in x and x["link"] != ""):
  83                                 return False
  84
  85                         import re
  86                         m = re.search("\[([A-Za-z0-9_]+)\]", x["title"])
  87                         if m is None:
  88                                 return False
  89
  90                         x["name"] = m.group(1)
  91                         return True
  92
  93                 def g(x):
  94                         return {
  95                                 "title":   x["title"],
  96                                 "author":  x["author"],
  97                                 "name":    x["name"],
  98                                 "topicId": x["topicId"],
  99                                 "link":    x["link"],
 100                         }
 101
 102                 krock_list_cache = [g(x) for x in list if h(x)]
 103                 krock_list_cache_by_name = {}
 104                 for x in krock_list_cache:
 105                         if not x["name"] in krock_list_cache_by_name:
 106                                 krock_list_cache_by_name[x["name"]] = []
 107
 108                         krock_list_cache_by_name[x["name"]].append(x)
 109
 110         return krock_list_cache, krock_list_cache_by_name
 111
 112 def findModInfo(author, name, link):
 113         list, lookup = getKrockList()
 114
 115         if name is not None and name in lookup:
 116                 if len(lookup[name]) == 1:
 117                         return lookup[name][0]
 118
 119                 for x in lookup[name]:
 120                         if x["author"] == author:
 121                                 return x
 122
 123         if link is not None and len(link) > 15:
 124                 for x in list:
 125                         if link in x["link"]:
 126                                 return x
 127
 128         return None
 129
 130
 131 def parseConf(string):
 132         retval = {}
 133         for line in string.split("\n"):
 134                 idx = line.find("=")
 135                 if idx > 0:
 136                         key   = line[:idx].strip()
 137                         value = line[idx+1:].strip()
 138                         retval[key] = value
 139
 140         return retval
 141
 142
 143 @celery.task()
 144 def getMeta(urlstr, author):
 145         url = urlparse(urlstr)
 146
 147         urlmaker = None
 148         if url.netloc == "github.com":
 149                 urlmaker = GithubURLMaker(url)
 150         else:
 151                 raise TaskError("Unsupported repo")
 152
 153         if not urlmaker.isValid():
 154                 raise TaskError("Error! Url maker not valid")
 155
 156         result = {}
 157
 158         result["repo"] = urlmaker.getRepoURL()
 159         result["issueTracker"] = urlmaker.getIssueTrackerURL()
 160
 161         try:
 162                 contents = urllib.request.urlopen(urlmaker.getModConfURL()).read().decode("utf-8")
 163                 conf = parseConf(contents)
 164                 for key in ["name", "description", "title"]:
 165                         try:
 166                                 result[key] = conf[key]
 167                         except KeyError:
 168                                 pass
 169         except HTTPError:
 170                 print("mod.conf does not exist")
 171
 172         if "name" in result:
 173                 result["title"] = result["name"].replace("_", " ").title()
 174
 175         if not "description" in result:
 176                 try:
 177                         contents = urllib.request.urlopen(urlmaker.getDescURL()).read().decode("utf-8")
 178                         result["description"] = contents.strip()
 179                 except HTTPError:
 180                         print("description.txt does not exist!")
 181
 182         if "description" in result:
 183                 desc = result["description"]
 184                 idx = desc.find(".") + 1
 185                 cutIdx = min(len(desc), 200 if idx < 5 else idx)
 186                 result["short_description"] = desc[:cutIdx]
 187
 188         info = findModInfo(author, result.get("name"), result["repo"])
 189         if info is not None:
 190                 result["forumId"] = info.get("topicId")
 191
 192         return result
 193
 194
 195 @celery.task()
 196 def makeVCSRelease(id, branch):
 197         release = PackageRelease.query.get(id)
 198
 199         if release is None:
 200                 raise TaskError("No such release!")
 201
 202         if release.package is None:
 203                 raise TaskError("No package attached to release")
 204
 205         url = urlparse(release.package.repo)
 206
 207         urlmaker = None
 208         if url.netloc == "github.com":
 209                 urlmaker = GithubURLMaker(url)
 210         else:
 211                 raise TaskError("Unsupported repo")
 212
 213         if not urlmaker.isValid():
 214                 raise TaskError("Invalid github repo URL")
 215
 216         commitsURL = urlmaker.getCommitsURL(branch)
 217         contents = urllib.request.urlopen(commitsURL).read().decode("utf-8")
 218         commits = json.loads(contents)
 219
 220         if len(commits) == 0 or not "sha" in commits[0]:
 221                 raise TaskError("No commits found")
 222
 223         release.url = urlmaker.getCommitDownload(commits[0]["sha"])
 224         print(release.url)
 225         release.task_id = None
 226         db.session.commit()
 227
 228         return release.url
 229
 230
 231 @celery.task()
 232 def importRepoScreenshot(id):
 233         package = Package.query.get(id)
 234         if package is None or package.soft_deleted:
 235                 raise Exception("Unexpected none package")
 236
 237         # Get URL Maker
 238         url = urlparse(package.repo)
 239         urlmaker = None
 240         if url.netloc == "github.com":
 241                 urlmaker = GithubURLMaker(url)
 242         else:
 243                 raise TaskError("Unsupported repo")
 244
 245         if not urlmaker.isValid():
 246                 raise TaskError("Error! Url maker not valid")
 247
 248         try:
 249                 filename = randomString(10) + ".png"
 250                 imagePath = os.path.join("app/public/uploads", filename)
 251                 print(imagePath)
 252                 urllib.request.urlretrieve(urlmaker.getScreenshotURL(), imagePath)
 253
 254                 ss = PackageScreenshot()
 255                 ss.package = package
 256                 ss.title   = "screenshot.png"
 257                 ss.url     = "/uploads/" + filename
 258                 db.session.add(ss)
 259                 db.session.commit()
 260
 261                 return "/uploads/" + filename
 262         except HTTPError:
 263                 print("screenshot.png does not exist")
 264
 265         return None