X-Git-Url: https://git.lizzy.rs/?a=blobdiff_plain;f=app%2Ftasks%2Fimporttasks.py;h=b66270268316d584f4f2b78519a412ced58472f3;hb=4a25435f7aec30228c2060dab497ca7e327f42b7;hp=ed435849cdd3c2c85d9612056df4facee8236b49;hpb=64f131ae27a7332245b5a4eb8e1e4879d7d99578;p=cheatdb.git diff --git a/app/tasks/importtasks.py b/app/tasks/importtasks.py index ed43584..b662702 100644 --- a/app/tasks/importtasks.py +++ b/app/tasks/importtasks.py @@ -17,15 +17,19 @@ import flask, json, os, git, tempfile, shutil, gitdb from git import GitCommandError +from git_archive_all import GitArchiver from flask_sqlalchemy import SQLAlchemy from urllib.error import HTTPError import urllib.request from urllib.parse import urlparse, quote_plus, urlsplit +from zipfile import ZipFile + from app import app from app.models import * from app.tasks import celery, TaskError from app.utils import randomString - +from .minetestcheck import build_tree, MinetestCheckError, ContentType +from .minetestcheck.config import parse_conf class GithubURLMaker: def __init__(self, url): @@ -126,173 +130,22 @@ def findModInfo(author, name, link): return None - -def parseConf(string): - retval = {} - for line in string.split("\n"): - idx = line.find("=") - if idx > 0: - key = line[:idx].strip() - value = line[idx+1:].strip() - retval[key] = value - - return retval - - -class PackageTreeNode: - def __init__(self, baseDir, author=None, repo=None, name=None): - print("Scanning " + baseDir) - self.baseDir = baseDir - self.author = author - self.name = name - self.repo = repo - self.meta = None - self.children = [] - - # Detect type - type = None - is_modpack = False - if os.path.isfile(baseDir + "/game.conf"): - type = PackageType.GAME - elif os.path.isfile(baseDir + "/init.lua"): - type = PackageType.MOD - elif os.path.isfile(baseDir + "/modpack.txt") or \ - os.path.isfile(baseDir + "/modpack.conf"): - type = PackageType.MOD - is_modpack = True - elif os.path.isdir(baseDir + "/mods"): - type = PackageType.GAME - elif os.listdir(baseDir) == []: - # probably a submodule - return - else: - raise TaskError("Unable to detect package type!") - - self.type = type - self.readMetaFiles() - - if self.type == PackageType.GAME: - self.addChildrenFromModDir(baseDir + "/mods") - elif is_modpack: - self.addChildrenFromModDir(baseDir) - - - def readMetaFiles(self): - result = {} - - # .conf file - try: - with open(self.baseDir + "/mod.conf", "r") as myfile: - conf = parseConf(myfile.read()) - for key in ["name", "description", "title", "depends", "optional_depends"]: - try: - result[key] = conf[key] - except KeyError: - pass - except IOError: - print("description.txt does not exist!") - - # description.txt - if not "description" in result: - try: - with open(self.baseDir + "/description.txt", "r") as myfile: - result["description"] = myfile.read() - except IOError: - print("description.txt does not exist!") - - # depends.txt - import re - pattern = re.compile("^([a-z0-9_]+)\??$") - if not "depends" in result and not "optional_depends" in result: - try: - with open(self.baseDir + "/depends.txt", "r") as myfile: - contents = myfile.read() - soft = [] - hard = [] - for line in contents.split("\n"): - line = line.strip() - if pattern.match(line): - if line[len(line) - 1] == "?": - soft.append( line[:-1]) - else: - hard.append(line) - - result["depends"] = hard - result["optional_depends"] = soft - - except IOError: - print("depends.txt does not exist!") - - else: - if "depends" in result: - result["depends"] = [x.strip() for x in result["depends"].split(",")] - if "optional_depends" in result: - result["optional_depends"] = [x.strip() for x in result["optional_depends"].split(",")] - - - # Calculate Title - if "name" in result and not "title" in result: - result["title"] = result["name"].replace("_", " ").title() - - # Calculate short description - if "description" in result: - desc = result["description"] - idx = desc.find(".") + 1 - cutIdx = min(len(desc), 200 if idx < 5 else idx) - result["short_description"] = desc[:cutIdx] - - # Get forum ID - info = findModInfo(self.author, result.get("name"), self.repo) - if info is not None: - result["forumId"] = info.get("topicId") - - if "name" in result: - self.name = result["name"] - del result["name"] - - self.meta = result - - def addChildrenFromModDir(self, dir): - for entry in next(os.walk(dir))[1]: - path = dir + "/" + entry - if not entry.startswith('.') and os.path.isdir(path): - self.children.append(PackageTreeNode(path, name=entry)) - - - def fold(self, attr, key=None, acc=None): - if acc is None: - acc = set() - - if self.meta is None: - return acc - - at = getattr(self, attr) - value = at if key is None else at.get(key) - - if isinstance(value, list): - acc |= set(value) - elif value is not None: - acc.add(value) - - for child in self.children: - child.fold(attr, key, acc) - - return acc - - def get(self, key): - return self.meta.get(key) - def generateGitURL(urlstr): scheme, netloc, path, query, frag = urlsplit(urlstr) return "http://:@" + netloc + path + query + +def getTempDir(): + return os.path.join(tempfile.gettempdir(), randomString(10)) + + # Clones a repo from an unvalidated URL. # Returns a tuple of path and repo on sucess. # Throws `TaskError` on failure. # Caller is responsible for deleting returned directory. def cloneRepo(urlstr, ref=None, recursive=False): - gitDir = tempfile.gettempdir() + "/" + randomString(10) + gitDir = getTempDir() err = None try: @@ -303,8 +156,14 @@ def cloneRepo(urlstr, ref=None, recursive=False): repo = git.Repo.clone_from(gitUrl, gitDir, \ progress=None, env=None, depth=1, recursive=recursive, kill_after_timeout=15) else: - repo = git.Repo.clone_from(gitUrl, gitDir, \ - progress=None, env=None, depth=1, recursive=recursive, kill_after_timeout=15, b=ref) + repo = git.Repo.init(gitDir) + origin = repo.create_remote("origin", url=gitUrl) + assert origin.exists() + origin.fetch() + origin.pull(ref) + + for submodule in repo.submodules: + submodule.update(init=True) return gitDir, repo @@ -322,7 +181,12 @@ def cloneRepo(urlstr, ref=None, recursive=False): @celery.task() def getMeta(urlstr, author): gitDir, _ = cloneRepo(urlstr, recursive=True) - tree = PackageTreeNode(gitDir, author=author, repo=urlstr) + + try: + tree = build_tree(gitDir, author=author, repo=urlstr) + except MinetestCheckError as err: + raise TaskError(str(err)) + shutil.rmtree(gitDir) result = {} @@ -371,6 +235,39 @@ def makeVCSReleaseFromGithub(id, branch, release, url): return release.url +@celery.task(bind=True) +def checkZipRelease(self, id, path): + release = PackageRelease.query.get(id) + if release is None: + raise TaskError("No such release!") + elif release.package is None: + raise TaskError("No package attached to release") + + temp = getTempDir() + try: + with ZipFile(path, 'r') as zip_ref: + zip_ref.extractall(temp) + + try: + tree = build_tree(temp, expected_type=ContentType[release.package.type.name], \ + author=release.package.author.username, name=release.package.name) + except MinetestCheckError as err: + if "Fails validation" not in release.title: + release.title += " (Fails validation)" + + release.task_id = self.request.id + release.approved = False + db.session.commit() + + raise TaskError(str(err)) + + release.task_id = None + release.approve(release.package.author) + db.session.commit() + + finally: + shutil.rmtree(temp) + @celery.task() def makeVCSRelease(id, branch): @@ -380,29 +277,37 @@ def makeVCSRelease(id, branch): elif release.package is None: raise TaskError("No package attached to release") - urlmaker = None - url = urlparse(release.package.repo) - if url.netloc == "github.com": - return makeVCSReleaseFromGithub(id, branch, release, url) - else: - gitDir, repo = cloneRepo(release.package.repo, ref=branch, recursive=True) + # url = urlparse(release.package.repo) + # if url.netloc == "github.com": + # return makeVCSReleaseFromGithub(id, branch, release, url) - try: - filename = randomString(10) + ".zip" - destPath = os.path.join("app/public/uploads", filename) - with open(destPath, "wb") as fp: - repo.archive(fp, format="zip") - - release.url = "/uploads/" + filename - release.task_id = None - release.commit_hash = repo.head.object.hexsha - release.approve(release.package.author) - print(release.url) - db.session.commit() + gitDir, repo = cloneRepo(release.package.repo, ref=branch, recursive=True) + + try: + tree = build_tree(gitDir, expected_type=ContentType[release.package.type.name], \ + author=release.package.author.username, name=release.package.name) + except MinetestCheckError as err: + raise TaskError(str(err)) - return release.url - finally: - shutil.rmtree(gitDir) + try: + filename = randomString(10) + ".zip" + destPath = os.path.join(app.config["UPLOAD_DIR"], filename) + + assert(not os.path.isfile(destPath)) + archiver = GitArchiver(force_sub=True, main_repo_abspath=gitDir) + archiver.create(destPath) + assert(os.path.isfile(destPath)) + + release.url = "/uploads/" + filename + release.task_id = None + release.commit_hash = repo.head.object.hexsha + release.approve(release.package.author) + print(release.url) + db.session.commit() + + return release.url + finally: + shutil.rmtree(gitDir) @celery.task() def importRepoScreenshot(id): @@ -424,7 +329,7 @@ def importRepoScreenshot(id): sourcePath = gitDir + "/screenshot." + ext if os.path.isfile(sourcePath): filename = randomString(10) + "." + ext - destPath = os.path.join("app/public/uploads", filename) + destPath = os.path.join(app.config["UPLOAD_DIR"], filename) shutil.copyfile(sourcePath, destPath) ss = PackageScreenshot() @@ -461,7 +366,7 @@ def getDepends(package): # try: contents = urllib.request.urlopen(urlmaker.getModConfURL()).read().decode("utf-8") - conf = parseConf(contents) + conf = parse_conf(contents) for key in ["depends", "optional_depends"]: try: result[key] = conf[key]