app/tasks/importtasks.py

   1 # ContentDB
   2 # Copyright (C) 2018  rubenwardy
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU General Public License as published by
   6 # the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU General Public License
  15 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
  16
  17
  18 import flask, json, os, git, tempfile, shutil, gitdb
  19 from git import GitCommandError
  20 from git_archive_all import GitArchiver
  21 from flask_sqlalchemy import SQLAlchemy
  22 from urllib.error import HTTPError
  23 import urllib.request
  24 from urllib.parse import urlparse, quote_plus, urlsplit
  25 from zipfile import ZipFile
  26
  27 from app import app
  28 from app.models import *
  29 from app.tasks import celery, TaskError
  30 from app.utils import randomString, getExtension
  31 from .minetestcheck import build_tree, MinetestCheckError, ContentType
  32 from .minetestcheck.config import parse_conf
  33
  34
  35 krock_list_cache = None
  36 krock_list_cache_by_name = None
  37 def getKrockList():
  38         global krock_list_cache
  39         global krock_list_cache_by_name
  40
  41         if krock_list_cache is None:
  42                 contents = urllib.request.urlopen("https://krock-works.uk.to/minetest/modList.php").read().decode("utf-8")
  43                 list = json.loads(contents)
  44
  45                 def h(x):
  46                         if not ("title"   in x and "author" in x and \
  47                                         "topicId" in x and "link"   in x and x["link"] != ""):
  48                                 return False
  49
  50                         import re
  51                         m = re.search("\[([A-Za-z0-9_]+)\]", x["title"])
  52                         if m is None:
  53                                 return False
  54
  55                         x["name"] = m.group(1)
  56                         return True
  57
  58                 def g(x):
  59                         return {
  60                                 "title":   x["title"],
  61                                 "author":  x["author"],
  62                                 "name": x["name"],
  63                                 "topicId": x["topicId"],
  64                                 "link": x["link"],
  65                         }
  66
  67                 krock_list_cache = [g(x) for x in list if h(x)]
  68                 krock_list_cache_by_name = {}
  69                 for x in krock_list_cache:
  70                         if not x["name"] in krock_list_cache_by_name:
  71                                 krock_list_cache_by_name[x["name"]] = []
  72
  73                         krock_list_cache_by_name[x["name"]].append(x)
  74
  75         return krock_list_cache, krock_list_cache_by_name
  76
  77
  78 def findModInfo(author, name, link):
  79         list, lookup = getKrockList()
  80
  81         if name is not None and name in lookup:
  82                 if len(lookup[name]) == 1:
  83                         return lookup[name][0]
  84
  85                 for x in lookup[name]:
  86                         if x["author"] == author:
  87                                 return x
  88
  89         if link is not None and len(link) > 15:
  90                 for x in list:
  91                         if link in x["link"]:
  92                                 return x
  93
  94         return None
  95
  96
  97 def generateGitURL(urlstr):
  98         scheme, netloc, path, query, frag = urlsplit(urlstr)
  99
 100         return "http://:@" + netloc + path + query
 101
 102
 103 def getTempDir():
 104         return os.path.join(tempfile.gettempdir(), randomString(10))
 105
 106
 107 # Clones a repo from an unvalidated URL.
 108 # Returns a tuple of path and repo on sucess.
 109 # Throws `TaskError` on failure.
 110 # Caller is responsible for deleting returned directory.
 111 def cloneRepo(urlstr, ref=None, recursive=False):
 112         gitDir = getTempDir()
 113
 114         err = None
 115         try:
 116                 gitUrl = generateGitURL(urlstr)
 117                 print("Cloning from " + gitUrl)
 118
 119                 if ref is None:
 120                         repo = git.Repo.clone_from(gitUrl, gitDir, \
 121                                         progress=None, env=None, depth=1, recursive=recursive, kill_after_timeout=15)
 122                 else:
 123                         repo = git.Repo.init(gitDir)
 124                         origin = repo.create_remote("origin", url=gitUrl)
 125                         assert origin.exists()
 126                         origin.fetch()
 127                         origin.pull(ref)
 128
 129                         for submodule in repo.submodules:
 130                                 submodule.update(init=True)
 131
 132                 return gitDir, repo
 133
 134         except GitCommandError as e:
 135                 # This is needed to stop the backtrace being weird
 136                 err = e.stderr
 137
 138         except gitdb.exc.BadName as e:
 139                 err = "Unable to find the reference " + (ref or "?") + "\n" + e.stderr
 140
 141         raise TaskError(err.replace("stderr: ", "") \
 142                         .replace("Cloning into '" + gitDir + "'...", "") \
 143                         .strip())
 144
 145
 146 @celery.task()
 147 def getMeta(urlstr, author):
 148         gitDir, _ = cloneRepo(urlstr, recursive=True)
 149
 150         try:
 151                 tree = build_tree(gitDir, author=author, repo=urlstr)
 152         except MinetestCheckError as err:
 153                 raise TaskError(str(err))
 154
 155         shutil.rmtree(gitDir)
 156
 157         result = {}
 158         result["name"] = tree.name
 159         result["provides"] = tree.getModNames()
 160         result["type"] = tree.type.name
 161
 162         for key in ["depends", "optional_depends"]:
 163                 result[key] = tree.fold("meta", key)
 164
 165         for key in ["title", "repo", "issueTracker", "forumId", "description", "short_description"]:
 166                 result[key] = tree.get(key)
 167
 168         for mod in result["provides"]:
 169                 result["depends"].discard(mod)
 170                 result["optional_depends"].discard(mod)
 171
 172         for key, value in result.items():
 173                 if isinstance(value, set):
 174                         result[key] = list(value)
 175
 176         return result
 177
 178
 179 def postReleaseCheckUpdate(self, release, path):
 180         try:
 181                 tree = build_tree(path, expected_type=ContentType[release.package.type.name], \
 182                         author=release.package.author.username, name=release.package.name)
 183
 184                 cache = {}
 185                 def getMetaPackages(names):
 186                         return [ MetaPackage.GetOrCreate(x, cache) for x in names ]
 187
 188                 provides = tree.getModNames()
 189
 190                 package = release.package
 191                 package.provides.clear()
 192                 package.provides.extend(getMetaPackages(tree.getModNames()))
 193
 194                 # Delete all meta package dependencies
 195                 package.dependencies.filter(Dependency.meta_package != None).delete()
 196
 197                 # Get raw dependencies
 198                 depends = tree.fold("meta", "depends")
 199                 optional_depends = tree.fold("meta", "optional_depends")
 200
 201                 # Filter out provides
 202                 for mod in provides:
 203                         depends.discard(mod)
 204                         optional_depends.discard(mod)
 205
 206                 # Raise error on unresolved game dependencies
 207                 if package.type == PackageType.GAME and len(depends) > 0:
 208                         deps = ", ".join(depends)
 209                         raise MinetestCheckError("Game has unresolved hard dependencies: " + deps)
 210
 211                 # Add dependencies
 212                 for meta in getMetaPackages(depends):
 213                         db.session.add(Dependency(package, meta=meta, optional=False))
 214
 215                 for meta in getMetaPackages(optional_depends):
 216                         db.session.add(Dependency(package, meta=meta, optional=True))
 217
 218                 # Update min/max
 219
 220                 if tree.meta.get("min_minetest_version"):
 221                         release.min_rel = MinetestRelease.get(tree.meta["min_minetest_version"], None)
 222
 223                 if tree.meta.get("max_minetest_version"):
 224                         release.max_rel = MinetestRelease.get(tree.meta["max_minetest_version"], None)
 225
 226                 return tree
 227
 228         except MinetestCheckError as err:
 229                 db.session.rollback()
 230
 231                 if "Fails validation" not in release.title:
 232                         release.title += " (Fails validation)"
 233
 234                 release.task_id = self.request.id
 235                 release.approved = False
 236                 db.session.commit()
 237
 238                 raise TaskError(str(err))
 239
 240
 241 @celery.task(bind=True)
 242 def updateMetaFromRelease(self, id, path):
 243         release = PackageRelease.query.get(id)
 244         if release is None:
 245                 raise TaskError("No such release!")
 246         elif release.package is None:
 247                 raise TaskError("No package attached to release")
 248
 249         print("updateMetaFromRelease: {} for {}/{}" \
 250                 .format(id, release.package.author.display_name, release.package.name))
 251
 252         temp = getTempDir()
 253         try:
 254                 with ZipFile(path, 'r') as zip_ref:
 255                         zip_ref.extractall(temp)
 256
 257                 postReleaseCheckUpdate(self, release, temp)
 258                 db.session.commit()
 259
 260         finally:
 261                 shutil.rmtree(temp)
 262
 263
 264 @celery.task(bind=True)
 265 def checkZipRelease(self, id, path):
 266         release = PackageRelease.query.get(id)
 267         if release is None:
 268                 raise TaskError("No such release!")
 269         elif release.package is None:
 270                 raise TaskError("No package attached to release")
 271
 272         temp = getTempDir()
 273         try:
 274                 with ZipFile(path, 'r') as zip_ref:
 275                         zip_ref.extractall(temp)
 276
 277                 postReleaseCheckUpdate(self, release, temp)
 278
 279                 release.task_id = None
 280                 release.approve(release.package.author)
 281                 db.session.commit()
 282
 283         finally:
 284                 shutil.rmtree(temp)
 285
 286
 287 @celery.task(bind=True)
 288 def makeVCSRelease(self, id, branch):
 289         release = PackageRelease.query.get(id)
 290         if release is None:
 291                 raise TaskError("No such release!")
 292         elif release.package is None:
 293                 raise TaskError("No package attached to release")
 294
 295         gitDir, repo = cloneRepo(release.package.repo, ref=branch, recursive=True)
 296
 297         postReleaseCheckUpdate(self, release, gitDir)
 298
 299         try:
 300                 filename = randomString(10) + ".zip"
 301                 destPath = os.path.join(app.config["UPLOAD_DIR"], filename)
 302
 303                 assert(not os.path.isfile(destPath))
 304                 archiver = GitArchiver(force_sub=True, main_repo_abspath=gitDir)
 305                 archiver.create(destPath)
 306                 assert(os.path.isfile(destPath))
 307
 308                 release.url         = "/uploads/" + filename
 309                 release.task_id     = None
 310                 release.commit_hash = repo.head.object.hexsha
 311                 release.approve(release.package.author)
 312                 db.session.commit()
 313
 314                 updateMetaFromRelease.delay(release.id, destPath)
 315
 316                 return release.url
 317         finally:
 318                 shutil.rmtree(gitDir)
 319
 320
 321 @celery.task()
 322 def importRepoScreenshot(id):
 323         package = Package.query.get(id)
 324         if package is None or package.soft_deleted:
 325                 raise Exception("Unexpected none package")
 326
 327         # Get URL Maker
 328         try:
 329                 gitDir, _ = cloneRepo(package.repo)
 330         except TaskError as e:
 331                 # ignore download errors
 332                 print(e)
 333                 return None
 334
 335         # Find and import screenshot
 336         try:
 337                 for ext in ["png", "jpg", "jpeg"]:
 338                         sourcePath = gitDir + "/screenshot." + ext
 339                         if os.path.isfile(sourcePath):
 340                                 filename = randomString(10) + "." + ext
 341                                 destPath = os.path.join(app.config["UPLOAD_DIR"], filename)
 342                                 shutil.copyfile(sourcePath, destPath)
 343
 344                                 ss = PackageScreenshot()
 345                                 ss.approved = True
 346                                 ss.package = package
 347                                 ss.title   = "screenshot.png"
 348                                 ss.url   = "/uploads/" + filename
 349                                 db.session.add(ss)
 350                                 db.session.commit()
 351
 352                                 return "/uploads/" + filename
 353         finally:
 354                 shutil.rmtree(gitDir)
 355
 356         print("screenshot.png does not exist")
 357         return None
 358
 359
 360 @celery.task(bind=True)
 361 def importForeignDownloads(self, id):
 362         release = PackageRelease.query.get(id)
 363         if release is None:
 364                 raise TaskError("No such release!")
 365         elif release.package is None:
 366                 raise TaskError("No package attached to release")
 367         elif not release.url.startswith("http"):
 368                 return
 369
 370         try:
 371                 ext = getExtension(release.url)
 372                 filename = randomString(10) + "." + ext
 373                 filepath = os.path.join(app.config["UPLOAD_DIR"], filename)
 374                 urllib.request.urlretrieve(release.url, filepath)
 375
 376                 release.url = "/uploads/" + filename
 377                 db.session.commit()
 378
 379         except urllib.error.URLError:
 380                 db.session.rollback()
 381                 release.task_id = self.request.id
 382                 release.approved = False
 383                 db.session.commit()