app/tasks/importtasks.py

   1 # ContentDB
   2 # Copyright (C) 2018  rubenwardy
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU General Public License as published by
   6 # the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU General Public License
  15 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
  16
  17
  18 import flask, json, os, git, tempfile, shutil, gitdb
  19 from git import GitCommandError
  20 from git_archive_all import GitArchiver
  21 from flask_sqlalchemy import SQLAlchemy
  22 from urllib.error import HTTPError
  23 import urllib.request
  24 from urllib.parse import urlparse, quote_plus, urlsplit
  25 from zipfile import ZipFile
  26
  27 from app import app
  28 from app.models import *
  29 from app.tasks import celery, TaskError
  30 from app.utils import randomString, getExtension
  31 from .minetestcheck import build_tree, MinetestCheckError, ContentType
  32 from .minetestcheck.config import parse_conf
  33
  34 krock_list_cache = None
  35 krock_list_cache_by_name = None
  36 def getKrockList():
  37         global krock_list_cache
  38         global krock_list_cache_by_name
  39
  40         if krock_list_cache is None:
  41                 contents = urllib.request.urlopen("https://krock-works.uk.to/minetest/modList.php").read().decode("utf-8")
  42                 list = json.loads(contents)
  43
  44                 def h(x):
  45                         if not ("title"   in x and "author" in x and \
  46                                         "topicId" in x and "link"   in x and x["link"] != ""):
  47                                 return False
  48
  49                         import re
  50                         m = re.search("\[([A-Za-z0-9_]+)\]", x["title"])
  51                         if m is None:
  52                                 return False
  53
  54                         x["name"] = m.group(1)
  55                         return True
  56
  57                 def g(x):
  58                         return {
  59                                 "title":   x["title"],
  60                                 "author":  x["author"],
  61                                 "name": x["name"],
  62                                 "topicId": x["topicId"],
  63                                 "link": x["link"],
  64                         }
  65
  66                 krock_list_cache = [g(x) for x in list if h(x)]
  67                 krock_list_cache_by_name = {}
  68                 for x in krock_list_cache:
  69                         if not x["name"] in krock_list_cache_by_name:
  70                                 krock_list_cache_by_name[x["name"]] = []
  71
  72                         krock_list_cache_by_name[x["name"]].append(x)
  73
  74         return krock_list_cache, krock_list_cache_by_name
  75
  76 def findModInfo(author, name, link):
  77         list, lookup = getKrockList()
  78
  79         if name is not None and name in lookup:
  80                 if len(lookup[name]) == 1:
  81                         return lookup[name][0]
  82
  83                 for x in lookup[name]:
  84                         if x["author"] == author:
  85                                 return x
  86
  87         if link is not None and len(link) > 15:
  88                 for x in list:
  89                         if link in x["link"]:
  90                                 return x
  91
  92         return None
  93
  94 def generateGitURL(urlstr):
  95         scheme, netloc, path, query, frag = urlsplit(urlstr)
  96
  97         return "http://:@" + netloc + path + query
  98
  99
 100 def getTempDir():
 101         return os.path.join(tempfile.gettempdir(), randomString(10))
 102
 103
 104 # Clones a repo from an unvalidated URL.
 105 # Returns a tuple of path and repo on sucess.
 106 # Throws `TaskError` on failure.
 107 # Caller is responsible for deleting returned directory.
 108 def cloneRepo(urlstr, ref=None, recursive=False):
 109         gitDir = getTempDir()
 110
 111         err = None
 112         try:
 113                 gitUrl = generateGitURL(urlstr)
 114                 print("Cloning from " + gitUrl)
 115
 116                 if ref is None:
 117                         repo = git.Repo.clone_from(gitUrl, gitDir, \
 118                                         progress=None, env=None, depth=1, recursive=recursive, kill_after_timeout=15)
 119                 else:
 120                         repo = git.Repo.init(gitDir)
 121                         origin = repo.create_remote("origin", url=gitUrl)
 122                         assert origin.exists()
 123                         origin.fetch()
 124                         origin.pull(ref)
 125
 126                         for submodule in repo.submodules:
 127                                 submodule.update(init=True)
 128
 129                 return gitDir, repo
 130
 131         except GitCommandError as e:
 132                 # This is needed to stop the backtrace being weird
 133                 err = e.stderr
 134
 135         except gitdb.exc.BadName as e:
 136                 err = "Unable to find the reference " + (ref or "?") + "\n" + e.stderr
 137
 138         raise TaskError(err.replace("stderr: ", "") \
 139                         .replace("Cloning into '" + gitDir + "'...", "") \
 140                         .strip())
 141
 142
 143 @celery.task(bind=True)
 144 def updateMetaFromRelease(self, id, path):
 145         release = PackageRelease.query.get(id)
 146         if release is None:
 147                 raise TaskError("No such release!")
 148         elif release.package is None:
 149                 raise TaskError("No package attached to release")
 150
 151         print("updateMetaFromRelease: {} for {}/{}" \
 152                 .format(id, release.package.author.display_name, release.package.name))
 153
 154         temp = getTempDir()
 155         try:
 156                 with ZipFile(path, 'r') as zip_ref:
 157                         zip_ref.extractall(temp)
 158
 159                 try:
 160                         tree = build_tree(temp, expected_type=ContentType[release.package.type.name], \
 161                                 author=release.package.author.username, name=release.package.name)
 162
 163                         cache = {}
 164                         def getMetaPackages(names):
 165                                 return [ MetaPackage.GetOrCreate(x, cache) for x in names ]
 166
 167                         provides = tree.getModNames()
 168
 169                         package = release.package
 170                         package.provides.clear()
 171                         package.provides.extend(getMetaPackages(tree.getModNames()))
 172
 173                         # Delete all meta package dependencies
 174                         package.dependencies.filter(Dependency.meta_package != None).delete()
 175
 176                         # Get raw dependencies
 177                         depends = tree.fold("meta", "depends")
 178                         optional_depends = tree.fold("meta", "optional_depends")
 179
 180                         # Filter out provides
 181                         for mod in provides:
 182                                 depends.discard(mod)
 183                                 optional_depends.discard(mod)
 184
 185                         # Add dependencies
 186
 187                         for meta in getMetaPackages(depends):
 188                                 db.session.add(Dependency(package, meta=meta, optional=False))
 189
 190                         for meta in getMetaPackages(optional_depends):
 191                                 db.session.add(Dependency(package, meta=meta, optional=True))
 192
 193
 194                         db.session.commit()
 195
 196                 except MinetestCheckError as err:
 197                         if "Fails validation" not in release.title:
 198                                 release.title += " (Fails validation)"
 199
 200                         release.task_id = self.request.id
 201                         release.approved = False
 202                         db.session.commit()
 203
 204                         raise TaskError(str(err))
 205
 206         finally:
 207                 shutil.rmtree(temp)
 208
 209
 210 @celery.task()
 211 def getMeta(urlstr, author):
 212         gitDir, _ = cloneRepo(urlstr, recursive=True)
 213
 214         try:
 215                 tree = build_tree(gitDir, author=author, repo=urlstr)
 216         except MinetestCheckError as err:
 217                 raise TaskError(str(err))
 218
 219         shutil.rmtree(gitDir)
 220
 221         result = {}
 222         result["name"] = tree.name
 223         result["provides"] = tree.getModNames()
 224         result["type"] = tree.type.name
 225
 226         for key in ["depends", "optional_depends"]:
 227                 result[key] = tree.fold("meta", key)
 228
 229         for key in ["title", "repo", "issueTracker", "forumId", "description", "short_description"]:
 230                 result[key] = tree.get(key)
 231
 232         for mod in result["provides"]:
 233                 result["depends"].discard(mod)
 234                 result["optional_depends"].discard(mod)
 235
 236         for key, value in result.items():
 237                 if isinstance(value, set):
 238                         result[key] = list(value)
 239
 240         return result
 241
 242
 243 @celery.task(bind=True)
 244 def checkZipRelease(self, id, path):
 245         release = PackageRelease.query.get(id)
 246         if release is None:
 247                 raise TaskError("No such release!")
 248         elif release.package is None:
 249                 raise TaskError("No package attached to release")
 250
 251         temp = getTempDir()
 252         try:
 253                 with ZipFile(path, 'r') as zip_ref:
 254                         zip_ref.extractall(temp)
 255
 256                 try:
 257                         tree = build_tree(temp, expected_type=ContentType[release.package.type.name], \
 258                                 author=release.package.author.username, name=release.package.name)
 259                 except MinetestCheckError as err:
 260                         if "Fails validation" not in release.title:
 261                                 release.title += " (Fails validation)"
 262
 263                         release.task_id = self.request.id
 264                         release.approved = False
 265                         db.session.commit()
 266
 267                         raise TaskError(str(err))
 268
 269                 release.task_id = None
 270                 release.approve(release.package.author)
 271                 db.session.commit()
 272
 273         finally:
 274                 shutil.rmtree(temp)
 275
 276
 277 @celery.task()
 278 def makeVCSRelease(id, branch):
 279         release = PackageRelease.query.get(id)
 280         if release is None:
 281                 raise TaskError("No such release!")
 282         elif release.package is None:
 283                 raise TaskError("No package attached to release")
 284
 285         gitDir, repo = cloneRepo(release.package.repo, ref=branch, recursive=True)
 286
 287         tree = None
 288         try:
 289                 tree = build_tree(gitDir, expected_type=ContentType[release.package.type.name], \
 290                         author=release.package.author.username, name=release.package.name)
 291         except MinetestCheckError as err:
 292                 raise TaskError(str(err))
 293
 294         try:
 295                 filename = randomString(10) + ".zip"
 296                 destPath = os.path.join(app.config["UPLOAD_DIR"], filename)
 297
 298                 assert(not os.path.isfile(destPath))
 299                 archiver = GitArchiver(force_sub=True, main_repo_abspath=gitDir)
 300                 archiver.create(destPath)
 301                 assert(os.path.isfile(destPath))
 302
 303                 release.url         = "/uploads/" + filename
 304                 release.task_id     = None
 305                 release.commit_hash = repo.head.object.hexsha
 306
 307                 if tree.meta.get("min_minetest_version"):
 308                         release.min_rel = MinetestRelease.get(tree.meta["min_minetest_version"], None)
 309
 310                 if tree.meta.get("max_minetest_version"):
 311                         release.max_rel = MinetestRelease.get(tree.meta["max_minetest_version"], None)
 312
 313                 release.approve(release.package.author)
 314                 db.session.commit()
 315
 316                 updateMetaFromRelease.delay(release.id, destPath)
 317
 318                 return release.url
 319         finally:
 320                 shutil.rmtree(gitDir)
 321
 322
 323 @celery.task()
 324 def importRepoScreenshot(id):
 325         package = Package.query.get(id)
 326         if package is None or package.soft_deleted:
 327                 raise Exception("Unexpected none package")
 328
 329         # Get URL Maker
 330         try:
 331                 gitDir, _ = cloneRepo(package.repo)
 332         except TaskError as e:
 333                 # ignore download errors
 334                 print(e)
 335                 return None
 336
 337         # Find and import screenshot
 338         try:
 339                 for ext in ["png", "jpg", "jpeg"]:
 340                         sourcePath = gitDir + "/screenshot." + ext
 341                         if os.path.isfile(sourcePath):
 342                                 filename = randomString(10) + "." + ext
 343                                 destPath = os.path.join(app.config["UPLOAD_DIR"], filename)
 344                                 shutil.copyfile(sourcePath, destPath)
 345
 346                                 ss = PackageScreenshot()
 347                                 ss.approved = True
 348                                 ss.package = package
 349                                 ss.title   = "screenshot.png"
 350                                 ss.url   = "/uploads/" + filename
 351                                 db.session.add(ss)
 352                                 db.session.commit()
 353
 354                                 return "/uploads/" + filename
 355         finally:
 356                 shutil.rmtree(gitDir)
 357
 358         print("screenshot.png does not exist")
 359         return None
 360
 361
 362 @celery.task(bind=True)
 363 def importForeignDownloads(self, id):
 364         release = PackageRelease.query.get(id)
 365         if release is None:
 366                 raise TaskError("No such release!")
 367         elif release.package is None:
 368                 raise TaskError("No package attached to release")
 369         elif not release.url.startswith("http"):
 370                 return
 371
 372         try:
 373                 ext = getExtension(release.url)
 374                 filename = randomString(10) + "." + ext
 375                 filepath = os.path.join(app.config["UPLOAD_DIR"], filename)
 376                 urllib.request.urlretrieve(release.url, filepath)
 377
 378                 release.url = "/uploads/" + filename
 379                 db.session.commit()
 380         except urllib.error.URLError:
 381                 release.task_id = self.request.id
 382                 release.approved = False
 383                 db.session.commit()