app/tasks/importtasks.py

   1 # ContentDB
   2 # Copyright (C) 2018  rubenwardy
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU General Public License as published by
   6 # the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU General Public License
  15 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
  16
  17
  18 import flask, json, os, git, tempfile, shutil, gitdb
  19 from git import GitCommandError
  20 from git_archive_all import GitArchiver
  21 from flask_sqlalchemy import SQLAlchemy
  22 from urllib.error import HTTPError
  23 import urllib.request
  24 from urllib.parse import urlparse, quote_plus, urlsplit
  25 from zipfile import ZipFile
  26
  27 from app import app
  28 from app.models import *
  29 from app.tasks import celery, TaskError
  30 from app.utils import randomString, getExtension
  31 from .minetestcheck import build_tree, MinetestCheckError, ContentType
  32 from .minetestcheck.config import parse_conf
  33
  34
  35 krock_list_cache = None
  36 krock_list_cache_by_name = None
  37 def getKrockList():
  38         global krock_list_cache
  39         global krock_list_cache_by_name
  40
  41         if krock_list_cache is None:
  42                 contents = urllib.request.urlopen("https://krock-works.uk.to/minetest/modList.php").read().decode("utf-8")
  43                 list = json.loads(contents)
  44
  45                 def h(x):
  46                         if not ("title"   in x and "author" in x and \
  47                                         "topicId" in x and "link"   in x and x["link"] != ""):
  48                                 return False
  49
  50                         import re
  51                         m = re.search("\[([A-Za-z0-9_]+)\]", x["title"])
  52                         if m is None:
  53                                 return False
  54
  55                         x["name"] = m.group(1)
  56                         return True
  57
  58                 def g(x):
  59                         return {
  60                                 "title":   x["title"],
  61                                 "author":  x["author"],
  62                                 "name": x["name"],
  63                                 "topicId": x["topicId"],
  64                                 "link": x["link"],
  65                         }
  66
  67                 krock_list_cache = [g(x) for x in list if h(x)]
  68                 krock_list_cache_by_name = {}
  69                 for x in krock_list_cache:
  70                         if not x["name"] in krock_list_cache_by_name:
  71                                 krock_list_cache_by_name[x["name"]] = []
  72
  73                         krock_list_cache_by_name[x["name"]].append(x)
  74
  75         return krock_list_cache, krock_list_cache_by_name
  76
  77
  78 def findModInfo(author, name, link):
  79         list, lookup = getKrockList()
  80
  81         if name is not None and name in lookup:
  82                 if len(lookup[name]) == 1:
  83                         return lookup[name][0]
  84
  85                 for x in lookup[name]:
  86                         if x["author"] == author:
  87                                 return x
  88
  89         if link is not None and len(link) > 15:
  90                 for x in list:
  91                         if link in x["link"]:
  92                                 return x
  93
  94         return None
  95
  96
  97 def generateGitURL(urlstr):
  98         scheme, netloc, path, query, frag = urlsplit(urlstr)
  99
 100         return "http://:@" + netloc + path + query
 101
 102
 103 def getTempDir():
 104         return os.path.join(tempfile.gettempdir(), randomString(10))
 105
 106
 107 # Clones a repo from an unvalidated URL.
 108 # Returns a tuple of path and repo on sucess.
 109 # Throws `TaskError` on failure.
 110 # Caller is responsible for deleting returned directory.
 111 def cloneRepo(urlstr, ref=None, recursive=False):
 112         gitDir = getTempDir()
 113
 114         err = None
 115         try:
 116                 gitUrl = generateGitURL(urlstr)
 117                 print("Cloning from " + gitUrl)
 118
 119                 if ref is None:
 120                         repo = git.Repo.clone_from(gitUrl, gitDir, \
 121                                         progress=None, env=None, depth=1, recursive=recursive, kill_after_timeout=15)
 122                 else:
 123                         repo = git.Repo.init(gitDir)
 124                         origin = repo.create_remote("origin", url=gitUrl)
 125                         assert origin.exists()
 126                         origin.fetch()
 127                         origin.pull(ref)
 128
 129                         for submodule in repo.submodules:
 130                                 submodule.update(init=True)
 131
 132                 return gitDir, repo
 133
 134         except GitCommandError as e:
 135                 # This is needed to stop the backtrace being weird
 136                 err = e.stderr
 137
 138         except gitdb.exc.BadName as e:
 139                 err = "Unable to find the reference " + (ref or "?") + "\n" + e.stderr
 140
 141         raise TaskError(err.replace("stderr: ", "") \
 142                         .replace("Cloning into '" + gitDir + "'...", "") \
 143                         .strip())
 144
 145
 146 @celery.task()
 147 def getMeta(urlstr, author):
 148         gitDir, _ = cloneRepo(urlstr, recursive=True)
 149
 150         try:
 151                 tree = build_tree(gitDir, author=author, repo=urlstr)
 152         except MinetestCheckError as err:
 153                 raise TaskError(str(err))
 154
 155         shutil.rmtree(gitDir)
 156
 157         result = {}
 158         result["name"] = tree.name
 159         result["provides"] = tree.getModNames()
 160         result["type"] = tree.type.name
 161
 162         for key in ["depends", "optional_depends"]:
 163                 result[key] = tree.fold("meta", key)
 164
 165         for key in ["title", "repo", "issueTracker", "forumId", "description", "short_description"]:
 166                 result[key] = tree.get(key)
 167
 168         for mod in result["provides"]:
 169                 result["depends"].discard(mod)
 170                 result["optional_depends"].discard(mod)
 171
 172         for key, value in result.items():
 173                 if isinstance(value, set):
 174                         result[key] = list(value)
 175
 176         return result
 177
 178
 179 def postReleaseCheckUpdate(self, release, path):
 180         try:
 181                 tree = build_tree(path, expected_type=ContentType[release.package.type.name], \
 182                         author=release.package.author.username, name=release.package.name)
 183
 184                 cache = {}
 185                 def getMetaPackages(names):
 186                         return [ MetaPackage.GetOrCreate(x, cache) for x in names ]
 187
 188                 provides = tree.getModNames()
 189
 190                 package = release.package
 191                 package.provides.clear()
 192                 package.provides.extend(getMetaPackages(tree.getModNames()))
 193
 194                 # Delete all meta package dependencies
 195                 package.dependencies.filter(Dependency.meta_package != None).delete()
 196
 197                 # Get raw dependencies
 198                 depends = tree.fold("meta", "depends")
 199                 optional_depends = tree.fold("meta", "optional_depends")
 200
 201                 # Filter out provides
 202                 for mod in provides:
 203                         depends.discard(mod)
 204                         optional_depends.discard(mod)
 205
 206                 # Raise error on unresolved game dependencies
 207                 if package.type == PackageType.GAME and len(depends) > 0:
 208                         deps = ", ".join(depends)
 209                         raise TaskError("Game has unresolved hard dependencies: " + deps)
 210
 211                 # Add dependencies
 212                 for meta in getMetaPackages(depends):
 213                         db.session.add(Dependency(package, meta=meta, optional=False))
 214
 215                 for meta in getMetaPackages(optional_depends):
 216                         db.session.add(Dependency(package, meta=meta, optional=True))
 217
 218                 # Update min/max
 219
 220                 if tree.meta.get("min_minetest_version"):
 221                         release.min_rel = MinetestRelease.get(tree.meta["min_minetest_version"], None)
 222
 223                 if tree.meta.get("max_minetest_version"):
 224                         release.max_rel = MinetestRelease.get(tree.meta["max_minetest_version"], None)
 225
 226                 return tree
 227
 228         except MinetestCheckError as err:
 229                 if "Fails validation" not in release.title:
 230                         release.title += " (Fails validation)"
 231
 232                 db.session.rollback()
 233                 release.task_id = self.request.id
 234                 release.approved = False
 235                 db.session.commit()
 236
 237                 raise TaskError(str(err))
 238
 239
 240 @celery.task(bind=True)
 241 def updateMetaFromRelease(self, id, path):
 242         release = PackageRelease.query.get(id)
 243         if release is None:
 244                 raise TaskError("No such release!")
 245         elif release.package is None:
 246                 raise TaskError("No package attached to release")
 247
 248         print("updateMetaFromRelease: {} for {}/{}" \
 249                 .format(id, release.package.author.display_name, release.package.name))
 250
 251         temp = getTempDir()
 252         try:
 253                 with ZipFile(path, 'r') as zip_ref:
 254                         zip_ref.extractall(temp)
 255
 256                 postReleaseCheckUpdate(self, release, temp)
 257                 db.session.commit()
 258
 259         finally:
 260                 shutil.rmtree(temp)
 261
 262
 263 @celery.task(bind=True)
 264 def checkZipRelease(self, id, path):
 265         release = PackageRelease.query.get(id)
 266         if release is None:
 267                 raise TaskError("No such release!")
 268         elif release.package is None:
 269                 raise TaskError("No package attached to release")
 270
 271         temp = getTempDir()
 272         try:
 273                 with ZipFile(path, 'r') as zip_ref:
 274                         zip_ref.extractall(temp)
 275
 276                 postReleaseCheckUpdate(self, release, temp)
 277
 278                 release.task_id = None
 279                 release.approve(release.package.author)
 280                 db.session.commit()
 281
 282         finally:
 283                 shutil.rmtree(temp)
 284
 285
 286 @celery.task(bind=True)
 287 def makeVCSRelease(self, id, branch):
 288         release = PackageRelease.query.get(id)
 289         if release is None:
 290                 raise TaskError("No such release!")
 291         elif release.package is None:
 292                 raise TaskError("No package attached to release")
 293
 294         gitDir, repo = cloneRepo(release.package.repo, ref=branch, recursive=True)
 295
 296         postReleaseCheckUpdate(self, release, gitDir)
 297
 298         try:
 299                 filename = randomString(10) + ".zip"
 300                 destPath = os.path.join(app.config["UPLOAD_DIR"], filename)
 301
 302                 assert(not os.path.isfile(destPath))
 303                 archiver = GitArchiver(force_sub=True, main_repo_abspath=gitDir)
 304                 archiver.create(destPath)
 305                 assert(os.path.isfile(destPath))
 306
 307                 release.url         = "/uploads/" + filename
 308                 release.task_id     = None
 309                 release.commit_hash = repo.head.object.hexsha
 310                 release.approve(release.package.author)
 311                 db.session.commit()
 312
 313                 updateMetaFromRelease.delay(release.id, destPath)
 314
 315                 return release.url
 316         finally:
 317                 shutil.rmtree(gitDir)
 318
 319
 320 @celery.task()
 321 def importRepoScreenshot(id):
 322         package = Package.query.get(id)
 323         if package is None or package.soft_deleted:
 324                 raise Exception("Unexpected none package")
 325
 326         # Get URL Maker
 327         try:
 328                 gitDir, _ = cloneRepo(package.repo)
 329         except TaskError as e:
 330                 # ignore download errors
 331                 print(e)
 332                 return None
 333
 334         # Find and import screenshot
 335         try:
 336                 for ext in ["png", "jpg", "jpeg"]:
 337                         sourcePath = gitDir + "/screenshot." + ext
 338                         if os.path.isfile(sourcePath):
 339                                 filename = randomString(10) + "." + ext
 340                                 destPath = os.path.join(app.config["UPLOAD_DIR"], filename)
 341                                 shutil.copyfile(sourcePath, destPath)
 342
 343                                 ss = PackageScreenshot()
 344                                 ss.approved = True
 345                                 ss.package = package
 346                                 ss.title   = "screenshot.png"
 347                                 ss.url   = "/uploads/" + filename
 348                                 db.session.add(ss)
 349                                 db.session.commit()
 350
 351                                 return "/uploads/" + filename
 352         finally:
 353                 shutil.rmtree(gitDir)
 354
 355         print("screenshot.png does not exist")
 356         return None
 357
 358
 359 @celery.task(bind=True)
 360 def importForeignDownloads(self, id):
 361         release = PackageRelease.query.get(id)
 362         if release is None:
 363                 raise TaskError("No such release!")
 364         elif release.package is None:
 365                 raise TaskError("No package attached to release")
 366         elif not release.url.startswith("http"):
 367                 return
 368
 369         try:
 370                 ext = getExtension(release.url)
 371                 filename = randomString(10) + "." + ext
 372                 filepath = os.path.join(app.config["UPLOAD_DIR"], filename)
 373                 urllib.request.urlretrieve(release.url, filepath)
 374
 375                 release.url = "/uploads/" + filename
 376                 db.session.commit()
 377
 378         except urllib.error.URLError:
 379                 db.session.rollback()
 380                 release.task_id = self.request.id
 381                 release.approved = False
 382                 db.session.commit()