app/tasks/importtasks.py

   1 # ContentDB
   2 # Copyright (C) 2018  rubenwardy
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU General Public License as published by
   6 # the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU General Public License
  15 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
  16
  17
  18 import flask, json, os, git, tempfile, shutil, gitdb
  19 from git import GitCommandError
  20 from git_archive_all import GitArchiver
  21 from flask_sqlalchemy import SQLAlchemy
  22 from urllib.error import HTTPError
  23 import urllib.request
  24 from urllib.parse import urlparse, quote_plus, urlsplit
  25 from zipfile import ZipFile
  26
  27 from app import app
  28 from app.models import *
  29 from app.tasks import celery, TaskError
  30 from app.utils import randomString, getExtension
  31 from .minetestcheck import build_tree, MinetestCheckError, ContentType
  32 from .minetestcheck.config import parse_conf
  33
  34 krock_list_cache = None
  35 krock_list_cache_by_name = None
  36 def getKrockList():
  37         global krock_list_cache
  38         global krock_list_cache_by_name
  39
  40         if krock_list_cache is None:
  41                 contents = urllib.request.urlopen("https://krock-works.uk.to/minetest/modList.php").read().decode("utf-8")
  42                 list = json.loads(contents)
  43
  44                 def h(x):
  45                         if not ("title"   in x and "author" in x and \
  46                                         "topicId" in x and "link"   in x and x["link"] != ""):
  47                                 return False
  48
  49                         import re
  50                         m = re.search("\[([A-Za-z0-9_]+)\]", x["title"])
  51                         if m is None:
  52                                 return False
  53
  54                         x["name"] = m.group(1)
  55                         return True
  56
  57                 def g(x):
  58                         return {
  59                                 "title":   x["title"],
  60                                 "author":  x["author"],
  61                                 "name": x["name"],
  62                                 "topicId": x["topicId"],
  63                                 "link": x["link"],
  64                         }
  65
  66                 krock_list_cache = [g(x) for x in list if h(x)]
  67                 krock_list_cache_by_name = {}
  68                 for x in krock_list_cache:
  69                         if not x["name"] in krock_list_cache_by_name:
  70                                 krock_list_cache_by_name[x["name"]] = []
  71
  72                         krock_list_cache_by_name[x["name"]].append(x)
  73
  74         return krock_list_cache, krock_list_cache_by_name
  75
  76 def findModInfo(author, name, link):
  77         list, lookup = getKrockList()
  78
  79         if name is not None and name in lookup:
  80                 if len(lookup[name]) == 1:
  81                         return lookup[name][0]
  82
  83                 for x in lookup[name]:
  84                         if x["author"] == author:
  85                                 return x
  86
  87         if link is not None and len(link) > 15:
  88                 for x in list:
  89                         if link in x["link"]:
  90                                 return x
  91
  92         return None
  93
  94 def generateGitURL(urlstr):
  95         scheme, netloc, path, query, frag = urlsplit(urlstr)
  96
  97         return "http://:@" + netloc + path + query
  98
  99
 100 def getTempDir():
 101         return os.path.join(tempfile.gettempdir(), randomString(10))
 102
 103
 104 # Clones a repo from an unvalidated URL.
 105 # Returns a tuple of path and repo on sucess.
 106 # Throws `TaskError` on failure.
 107 # Caller is responsible for deleting returned directory.
 108 def cloneRepo(urlstr, ref=None, recursive=False):
 109         gitDir = getTempDir()
 110
 111         err = None
 112         try:
 113                 gitUrl = generateGitURL(urlstr)
 114                 print("Cloning from " + gitUrl)
 115
 116                 if ref is None:
 117                         repo = git.Repo.clone_from(gitUrl, gitDir, \
 118                                         progress=None, env=None, depth=1, recursive=recursive, kill_after_timeout=15)
 119                 else:
 120                         repo = git.Repo.init(gitDir)
 121                         origin = repo.create_remote("origin", url=gitUrl)
 122                         assert origin.exists()
 123                         origin.fetch()
 124                         origin.pull(ref)
 125
 126                         for submodule in repo.submodules:
 127                                 submodule.update(init=True)
 128
 129                 return gitDir, repo
 130
 131         except GitCommandError as e:
 132                 # This is needed to stop the backtrace being weird
 133                 err = e.stderr
 134
 135         except gitdb.exc.BadName as e:
 136                 err = "Unable to find the reference " + (ref or "?") + "\n" + e.stderr
 137
 138         raise TaskError(err.replace("stderr: ", "") \
 139                         .replace("Cloning into '" + gitDir + "'...", "") \
 140                         .strip())
 141
 142
 143 @celery.task(bind=True)
 144 def updateMetaFromRelease(self, id, path):
 145         release = PackageRelease.query.get(id)
 146         if release is None:
 147                 raise TaskError("No such release!")
 148         elif release.package is None:
 149                 raise TaskError("No package attached to release")
 150
 151         temp = getTempDir()
 152         try:
 153                 with ZipFile(path, 'r') as zip_ref:
 154                         zip_ref.extractall(temp)
 155
 156                 try:
 157                         tree = build_tree(temp, expected_type=ContentType[release.package.type.name], \
 158                                 author=release.package.author.username, name=release.package.name)
 159
 160                         cache = {}
 161                         def getMetaPackages(names):
 162                                 return [ MetaPackage.GetOrCreate(x, cache) for x in names ]
 163
 164                         provides = getMetaPackages(tree.getModNames())
 165
 166                         package = release.package
 167                         package.provides.clear()
 168                         package.provides.extend(provides)
 169
 170                         for dep in package.dependencies:
 171                                 if dep.meta_package:
 172                                         db.session.delete(dep)
 173
 174                         for meta in getMetaPackages(tree.fold("meta", "depends")):
 175                                 db.session.add(Dependency(package, meta=meta, optional=False))
 176
 177                         for meta in getMetaPackages(tree.fold("meta", "optional_depends")):
 178                                 db.session.add(Dependency(package, meta=meta, optional=True))
 179
 180                         db.session.commit()
 181
 182                 except MinetestCheckError as err:
 183                         if "Fails validation" not in release.title:
 184                                 release.title += " (Fails validation)"
 185
 186                         release.task_id = self.request.id
 187                         release.approved = False
 188                         db.session.commit()
 189
 190                         raise TaskError(str(err))
 191
 192         finally:
 193                 shutil.rmtree(temp)
 194
 195
 196 @celery.task()
 197 def getMeta(urlstr, author):
 198         gitDir, _ = cloneRepo(urlstr, recursive=True)
 199
 200         try:
 201                 tree = build_tree(gitDir, author=author, repo=urlstr)
 202         except MinetestCheckError as err:
 203                 raise TaskError(str(err))
 204
 205         shutil.rmtree(gitDir)
 206
 207         result = {}
 208         result["name"] = tree.name
 209         result["provides"] = tree.getModNames()
 210         result["type"] = tree.type.name
 211
 212         for key in ["depends", "optional_depends"]:
 213                 result[key] = tree.fold("meta", key)
 214
 215         for key in ["title", "repo", "issueTracker", "forumId", "description", "short_description"]:
 216                 result[key] = tree.get(key)
 217
 218         for mod in result["provides"]:
 219                 result["depends"].discard(mod)
 220                 result["optional_depends"].discard(mod)
 221
 222         for key, value in result.items():
 223                 if isinstance(value, set):
 224                         result[key] = list(value)
 225
 226         return result
 227
 228
 229 @celery.task(bind=True)
 230 def checkZipRelease(self, id, path):
 231         release = PackageRelease.query.get(id)
 232         if release is None:
 233                 raise TaskError("No such release!")
 234         elif release.package is None:
 235                 raise TaskError("No package attached to release")
 236
 237         temp = getTempDir()
 238         try:
 239                 with ZipFile(path, 'r') as zip_ref:
 240                         zip_ref.extractall(temp)
 241
 242                 try:
 243                         tree = build_tree(temp, expected_type=ContentType[release.package.type.name], \
 244                                 author=release.package.author.username, name=release.package.name)
 245                 except MinetestCheckError as err:
 246                         if "Fails validation" not in release.title:
 247                                 release.title += " (Fails validation)"
 248
 249                         release.task_id = self.request.id
 250                         release.approved = False
 251                         db.session.commit()
 252
 253                         raise TaskError(str(err))
 254
 255                 release.task_id = None
 256                 release.approve(release.package.author)
 257                 db.session.commit()
 258
 259         finally:
 260                 shutil.rmtree(temp)
 261
 262
 263 @celery.task()
 264 def makeVCSRelease(id, branch):
 265         release = PackageRelease.query.get(id)
 266         if release is None:
 267                 raise TaskError("No such release!")
 268         elif release.package is None:
 269                 raise TaskError("No package attached to release")
 270
 271         gitDir, repo = cloneRepo(release.package.repo, ref=branch, recursive=True)
 272
 273         tree = None
 274         try:
 275                 tree = build_tree(gitDir, expected_type=ContentType[release.package.type.name], \
 276                         author=release.package.author.username, name=release.package.name)
 277         except MinetestCheckError as err:
 278                 raise TaskError(str(err))
 279
 280         try:
 281                 filename = randomString(10) + ".zip"
 282                 destPath = os.path.join(app.config["UPLOAD_DIR"], filename)
 283
 284                 assert(not os.path.isfile(destPath))
 285                 archiver = GitArchiver(force_sub=True, main_repo_abspath=gitDir)
 286                 archiver.create(destPath)
 287                 assert(os.path.isfile(destPath))
 288
 289                 release.url         = "/uploads/" + filename
 290                 release.task_id     = None
 291                 release.commit_hash = repo.head.object.hexsha
 292
 293                 if tree.meta.get("min_minetest_version"):
 294                         release.min_rel = MinetestRelease.get(tree.meta["min_minetest_version"], None)
 295
 296                 if tree.meta.get("max_minetest_version"):
 297                         release.max_rel = MinetestRelease.get(tree.meta["max_minetest_version"], None)
 298
 299                 release.approve(release.package.author)
 300                 db.session.commit()
 301
 302                 return release.url
 303         finally:
 304                 shutil.rmtree(gitDir)
 305
 306
 307 @celery.task()
 308 def importRepoScreenshot(id):
 309         package = Package.query.get(id)
 310         if package is None or package.soft_deleted:
 311                 raise Exception("Unexpected none package")
 312
 313         # Get URL Maker
 314         try:
 315                 gitDir, _ = cloneRepo(package.repo)
 316         except TaskError as e:
 317                 # ignore download errors
 318                 print(e)
 319                 return None
 320
 321         # Find and import screenshot
 322         try:
 323                 for ext in ["png", "jpg", "jpeg"]:
 324                         sourcePath = gitDir + "/screenshot." + ext
 325                         if os.path.isfile(sourcePath):
 326                                 filename = randomString(10) + "." + ext
 327                                 destPath = os.path.join(app.config["UPLOAD_DIR"], filename)
 328                                 shutil.copyfile(sourcePath, destPath)
 329
 330                                 ss = PackageScreenshot()
 331                                 ss.approved = True
 332                                 ss.package = package
 333                                 ss.title   = "screenshot.png"
 334                                 ss.url   = "/uploads/" + filename
 335                                 db.session.add(ss)
 336                                 db.session.commit()
 337
 338                                 return "/uploads/" + filename
 339         finally:
 340                 shutil.rmtree(gitDir)
 341
 342         print("screenshot.png does not exist")
 343         return None
 344
 345
 346 @celery.task(bind=True)
 347 def importForeignDownloads(self, id):
 348         release = PackageRelease.query.get(id)
 349         if release is None:
 350                 raise TaskError("No such release!")
 351         elif release.package is None:
 352                 raise TaskError("No package attached to release")
 353         elif not release.url.startswith("http"):
 354                 return
 355
 356         try:
 357                 ext = getExtension(release.url)
 358                 filename = randomString(10) + "." + ext
 359                 filepath = os.path.join(app.config["UPLOAD_DIR"], filename)
 360                 urllib.request.urlretrieve(release.url, filepath)
 361
 362                 release.url = "/uploads/" + filename
 363                 db.session.commit()
 364         except urllib.error.URLError:
 365                 release.task_id = self.request.id
 366                 release.approved = False
 367                 db.session.commit()