app/tasks/importtasks.py

   1 # ContentDB
   2 # Copyright (C) 2018  rubenwardy
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU General Public License as published by
   6 # the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU General Public License
  15 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
  16
  17
  18 import flask, json, os, git, tempfile, shutil, gitdb
  19 from git import GitCommandError
  20 from git_archive_all import GitArchiver
  21 from flask_sqlalchemy import SQLAlchemy
  22 from urllib.error import HTTPError
  23 import urllib.request
  24 from urllib.parse import urlparse, quote_plus, urlsplit
  25 from zipfile import ZipFile
  26
  27 from app import app
  28 from app.models import *
  29 from app.tasks import celery, TaskError
  30 from app.utils import randomString, getExtension
  31 from .minetestcheck import build_tree, MinetestCheckError, ContentType
  32 from .minetestcheck.config import parse_conf
  33
  34
  35 krock_list_cache = None
  36 krock_list_cache_by_name = None
  37 def getKrockList():
  38         global krock_list_cache
  39         global krock_list_cache_by_name
  40
  41         if krock_list_cache is None:
  42                 contents = urllib.request.urlopen("https://krock-works.uk.to/minetest/modList.php").read().decode("utf-8")
  43                 list = json.loads(contents)
  44
  45                 def h(x):
  46                         if not ("title"   in x and "author" in x and \
  47                                         "topicId" in x and "link"   in x and x["link"] != ""):
  48                                 return False
  49
  50                         import re
  51                         m = re.search("\[([A-Za-z0-9_]+)\]", x["title"])
  52                         if m is None:
  53                                 return False
  54
  55                         x["name"] = m.group(1)
  56                         return True
  57
  58                 def g(x):
  59                         return {
  60                                 "title":   x["title"],
  61                                 "author":  x["author"],
  62                                 "name": x["name"],
  63                                 "topicId": x["topicId"],
  64                                 "link": x["link"],
  65                         }
  66
  67                 krock_list_cache = [g(x) for x in list if h(x)]
  68                 krock_list_cache_by_name = {}
  69                 for x in krock_list_cache:
  70                         if not x["name"] in krock_list_cache_by_name:
  71                                 krock_list_cache_by_name[x["name"]] = []
  72
  73                         krock_list_cache_by_name[x["name"]].append(x)
  74
  75         return krock_list_cache, krock_list_cache_by_name
  76
  77
  78 def findModInfo(author, name, link):
  79         list, lookup = getKrockList()
  80
  81         if name is not None and name in lookup:
  82                 if len(lookup[name]) == 1:
  83                         return lookup[name][0]
  84
  85                 for x in lookup[name]:
  86                         if x["author"] == author:
  87                                 return x
  88
  89         if link is not None and len(link) > 15:
  90                 for x in list:
  91                         if link in x["link"]:
  92                                 return x
  93
  94         return None
  95
  96
  97 def generateGitURL(urlstr):
  98         scheme, netloc, path, query, frag = urlsplit(urlstr)
  99
 100         return "http://:@" + netloc + path + query
 101
 102
 103 def getTempDir():
 104         return os.path.join(tempfile.gettempdir(), randomString(10))
 105
 106
 107 # Clones a repo from an unvalidated URL.
 108 # Returns a tuple of path and repo on sucess.
 109 # Throws `TaskError` on failure.
 110 # Caller is responsible for deleting returned directory.
 111 def cloneRepo(urlstr, ref=None, recursive=False):
 112         gitDir = getTempDir()
 113
 114         err = None
 115         try:
 116                 gitUrl = generateGitURL(urlstr)
 117                 print("Cloning from " + gitUrl)
 118
 119                 if ref is None:
 120                         repo = git.Repo.clone_from(gitUrl, gitDir, \
 121                                         progress=None, env=None, depth=1, recursive=recursive, kill_after_timeout=15)
 122                 else:
 123                         repo = git.Repo.init(gitDir)
 124                         origin = repo.create_remote("origin", url=gitUrl)
 125                         assert origin.exists()
 126                         origin.fetch()
 127                         origin.pull(ref)
 128
 129                         for submodule in repo.submodules:
 130                                 submodule.update(init=True)
 131
 132                 return gitDir, repo
 133
 134         except GitCommandError as e:
 135                 # This is needed to stop the backtrace being weird
 136                 err = e.stderr
 137
 138         except gitdb.exc.BadName as e:
 139                 err = "Unable to find the reference " + (ref or "?") + "\n" + e.stderr
 140
 141         raise TaskError(err.replace("stderr: ", "") \
 142                         .replace("Cloning into '" + gitDir + "'...", "") \
 143                         .strip())
 144
 145
 146 @celery.task()
 147 def getMeta(urlstr, author):
 148         gitDir, _ = cloneRepo(urlstr, recursive=True)
 149
 150         try:
 151                 tree = build_tree(gitDir, author=author, repo=urlstr)
 152         except MinetestCheckError as err:
 153                 raise TaskError(str(err))
 154
 155         shutil.rmtree(gitDir)
 156
 157         result = {}
 158         result["name"] = tree.name
 159         result["provides"] = tree.getModNames()
 160         result["type"] = tree.type.name
 161
 162         for key in ["depends", "optional_depends"]:
 163                 result[key] = tree.fold("meta", key)
 164
 165         for key in ["title", "repo", "issueTracker", "forumId", "description", "short_description"]:
 166                 result[key] = tree.get(key)
 167
 168         for mod in result["provides"]:
 169                 result["depends"].discard(mod)
 170                 result["optional_depends"].discard(mod)
 171
 172         for key, value in result.items():
 173                 if isinstance(value, set):
 174                         result[key] = list(value)
 175
 176         return result
 177
 178
 179 def postReleaseCheckUpdate(self, release, path):
 180         try:
 181                 tree = build_tree(path, expected_type=ContentType[release.package.type.name], \
 182                         author=release.package.author.username, name=release.package.name)
 183
 184                 cache = {}
 185                 def getMetaPackages(names):
 186                         return [ MetaPackage.GetOrCreate(x, cache) for x in names ]
 187
 188                 provides = tree.getModNames()
 189
 190                 package = release.package
 191                 package.provides.clear()
 192                 package.provides.extend(getMetaPackages(tree.getModNames()))
 193
 194                 # Delete all meta package dependencies
 195                 package.dependencies.filter(Dependency.meta_package != None).delete()
 196
 197                 # Get raw dependencies
 198                 depends = tree.fold("meta", "depends")
 199                 optional_depends = tree.fold("meta", "optional_depends")
 200
 201                 # Filter out provides
 202                 for mod in provides:
 203                         depends.discard(mod)
 204                         optional_depends.discard(mod)
 205
 206                 # Add dependencies
 207
 208                 for meta in getMetaPackages(depends):
 209                         db.session.add(Dependency(package, meta=meta, optional=False))
 210
 211                 for meta in getMetaPackages(optional_depends):
 212                         db.session.add(Dependency(package, meta=meta, optional=True))
 213
 214                 # Update min/max
 215
 216                 if tree.meta.get("min_minetest_version"):
 217                         release.min_rel = MinetestRelease.get(tree.meta["min_minetest_version"], None)
 218
 219                 if tree.meta.get("max_minetest_version"):
 220                         release.max_rel = MinetestRelease.get(tree.meta["max_minetest_version"], None)
 221
 222                 return tree
 223
 224         except MinetestCheckError as err:
 225                 if "Fails validation" not in release.title:
 226                         release.title += " (Fails validation)"
 227
 228                 db.session.rollback()
 229                 release.task_id = self.request.id
 230                 release.approved = False
 231                 db.session.commit()
 232
 233                 raise TaskError(str(err))
 234
 235
 236 @celery.task(bind=True)
 237 def updateMetaFromRelease(self, id, path):
 238         release = PackageRelease.query.get(id)
 239         if release is None:
 240                 raise TaskError("No such release!")
 241         elif release.package is None:
 242                 raise TaskError("No package attached to release")
 243
 244         print("updateMetaFromRelease: {} for {}/{}" \
 245                 .format(id, release.package.author.display_name, release.package.name))
 246
 247         temp = getTempDir()
 248         try:
 249                 with ZipFile(path, 'r') as zip_ref:
 250                         zip_ref.extractall(temp)
 251
 252                 postReleaseCheckUpdate(self, release, temp)
 253                 db.session.commit()
 254
 255         finally:
 256                 shutil.rmtree(temp)
 257
 258
 259 @celery.task(bind=True)
 260 def checkZipRelease(self, id, path):
 261         release = PackageRelease.query.get(id)
 262         if release is None:
 263                 raise TaskError("No such release!")
 264         elif release.package is None:
 265                 raise TaskError("No package attached to release")
 266
 267         temp = getTempDir()
 268         try:
 269                 with ZipFile(path, 'r') as zip_ref:
 270                         zip_ref.extractall(temp)
 271
 272                 postReleaseCheckUpdate(self, release, temp)
 273
 274                 release.task_id = None
 275                 release.approve(release.package.author)
 276                 db.session.commit()
 277
 278         finally:
 279                 shutil.rmtree(temp)
 280
 281
 282 @celery.task(bind=True)
 283 def makeVCSRelease(self, id, branch):
 284         release = PackageRelease.query.get(id)
 285         if release is None:
 286                 raise TaskError("No such release!")
 287         elif release.package is None:
 288                 raise TaskError("No package attached to release")
 289
 290         gitDir, repo = cloneRepo(release.package.repo, ref=branch, recursive=True)
 291
 292         postReleaseCheckUpdate(self, release, gitDir)
 293
 294         try:
 295                 filename = randomString(10) + ".zip"
 296                 destPath = os.path.join(app.config["UPLOAD_DIR"], filename)
 297
 298                 assert(not os.path.isfile(destPath))
 299                 archiver = GitArchiver(force_sub=True, main_repo_abspath=gitDir)
 300                 archiver.create(destPath)
 301                 assert(os.path.isfile(destPath))
 302
 303                 release.url         = "/uploads/" + filename
 304                 release.task_id     = None
 305                 release.commit_hash = repo.head.object.hexsha
 306                 release.approve(release.package.author)
 307                 db.session.commit()
 308
 309                 updateMetaFromRelease.delay(release.id, destPath)
 310
 311                 return release.url
 312         finally:
 313                 shutil.rmtree(gitDir)
 314
 315
 316 @celery.task()
 317 def importRepoScreenshot(id):
 318         package = Package.query.get(id)
 319         if package is None or package.soft_deleted:
 320                 raise Exception("Unexpected none package")
 321
 322         # Get URL Maker
 323         try:
 324                 gitDir, _ = cloneRepo(package.repo)
 325         except TaskError as e:
 326                 # ignore download errors
 327                 print(e)
 328                 return None
 329
 330         # Find and import screenshot
 331         try:
 332                 for ext in ["png", "jpg", "jpeg"]:
 333                         sourcePath = gitDir + "/screenshot." + ext
 334                         if os.path.isfile(sourcePath):
 335                                 filename = randomString(10) + "." + ext
 336                                 destPath = os.path.join(app.config["UPLOAD_DIR"], filename)
 337                                 shutil.copyfile(sourcePath, destPath)
 338
 339                                 ss = PackageScreenshot()
 340                                 ss.approved = True
 341                                 ss.package = package
 342                                 ss.title   = "screenshot.png"
 343                                 ss.url   = "/uploads/" + filename
 344                                 db.session.add(ss)
 345                                 db.session.commit()
 346
 347                                 return "/uploads/" + filename
 348         finally:
 349                 shutil.rmtree(gitDir)
 350
 351         print("screenshot.png does not exist")
 352         return None
 353
 354
 355 @celery.task(bind=True)
 356 def importForeignDownloads(self, id):
 357         release = PackageRelease.query.get(id)
 358         if release is None:
 359                 raise TaskError("No such release!")
 360         elif release.package is None:
 361                 raise TaskError("No package attached to release")
 362         elif not release.url.startswith("http"):
 363                 return
 364
 365         try:
 366                 ext = getExtension(release.url)
 367                 filename = randomString(10) + "." + ext
 368                 filepath = os.path.join(app.config["UPLOAD_DIR"], filename)
 369                 urllib.request.urlretrieve(release.url, filepath)
 370
 371                 release.url = "/uploads/" + filename
 372                 db.session.commit()
 373
 374         except urllib.error.URLError:
 375                 db.session.rollback()
 376                 release.task_id = self.request.id
 377                 release.approved = False
 378                 db.session.commit()