X-Git-Url: https://git.lizzy.rs/?a=blobdiff_plain;f=app%2Ftasks%2Fimporttasks.py;h=1361afe384652b96c7b628956fa76406d721d552;hb=a123f422911c7c543b35ca405ecb4524f905a1a9;hp=db992b3c7fd4e60b54ea114c7ed99dd785d4db4f;hpb=53df124973c15b90e070f9225fcbeccdb0e64e55;p=cheatdb.git

diff --git a/app/tasks/importtasks.py b/app/tasks/importtasks.py
index db992b3..1361afe 100644
--- a/app/tasks/importtasks.py
+++ b/app/tasks/importtasks.py
@@ -15,57 +15,21 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 
-import flask, json, os
-from flask.ext.sqlalchemy import SQLAlchemy
+import flask, json, os, git, tempfile, shutil, gitdb
+from git import GitCommandError
+from git_archive_all import GitArchiver
+from flask_sqlalchemy import SQLAlchemy
 from urllib.error import HTTPError
 import urllib.request
-from urllib.parse import urlparse, quote_plus
+from urllib.parse import urlparse, quote_plus, urlsplit
+from zipfile import ZipFile
+
 from app import app
 from app.models import *
 from app.tasks import celery, TaskError
 from app.utils import randomString
-
-class GithubURLMaker:
-	def __init__(self, url):
-		# Rewrite path
-		import re
-		m = re.search("^\/([^\/]+)\/([^\/]+)\/?$", url.path)
-		if m is None:
-			return
-
-		user = m.group(1)
-		repo = m.group(2).replace(".git", "")
-		self.baseUrl = "https://raw.githubusercontent.com/{}/{}/master" \
-				.format(user, repo)
-		self.user = user
-		self.repo = repo
-
-	def isValid(self):
-		return self.baseUrl is not None
-
-	def getRepoURL(self):
-		return "https://github.com/{}/{}".format(self.user, self.repo)
-
-	def getIssueTrackerURL(self):
-		return "https://github.com/{}/{}/issues/".format(self.user, self.repo)
-
-	def getModConfURL(self):
-		return self.baseUrl + "/mod.conf"
-
-	def getDescURL(self):
-		return self.baseUrl + "/description.txt"
-
-	def getScreenshotURL(self):
-		return self.baseUrl + "/screenshot.png"
-
-	def getCommitsURL(self, branch):
-		return "https://api.github.com/repos/{}/{}/commits?sha={}" \
-				.format(self.user, self.repo, urllib.parse.quote_plus(branch))
-
-	def getCommitDownload(self, commit):
-		return "https://github.com/{}/{}/archive/{}.zip" \
-				.format(self.user, self.repo, commit)
-
+from .minetestcheck import build_tree, MinetestCheckError, ContentType
+from .minetestcheck.config import parse_conf
 
 krock_list_cache = None
 krock_list_cache_by_name = None
@@ -74,7 +38,7 @@ def getKrockList():
 	global krock_list_cache_by_name
 
 	if krock_list_cache is None:
-		contents = urllib.request.urlopen("http://krock-works.16mb.com/MTstuff/modList.php").read().decode("utf-8")
+		contents = urllib.request.urlopen("https://krock-works.uk.to/minetest/modList.php").read().decode("utf-8")
 		list = json.loads(contents)
 
 		def h(x):
@@ -94,9 +58,9 @@ def getKrockList():
 			return {
 				"title":   x["title"],
 				"author":  x["author"],
-				"name":    x["name"],
+				"name":	x["name"],
 				"topicId": x["topicId"],
-				"link":    x["link"],
+				"link":	x["link"],
 			}
 
 		krock_list_cache = [g(x) for x in list if h(x)]
@@ -127,105 +91,217 @@ def findModInfo(author, name, link):
 
 	return None
 
+def generateGitURL(urlstr):
+	scheme, netloc, path, query, frag = urlsplit(urlstr)
+
+	return "http://:@" + netloc + path + query
+
+
+def getTempDir():
+	return os.path.join(tempfile.gettempdir(), randomString(10))
+
+
+# Clones a repo from an unvalidated URL.
+# Returns a tuple of path and repo on sucess.
+# Throws `TaskError` on failure.
+# Caller is responsible for deleting returned directory.
+def cloneRepo(urlstr, ref=None, recursive=False):
+	gitDir = getTempDir()
+
+	err = None
+	try:
+		gitUrl = generateGitURL(urlstr)
+		print("Cloning from " + gitUrl)
+
+		if ref is None:
+			repo = git.Repo.clone_from(gitUrl, gitDir, \
+					progress=None, env=None, depth=1, recursive=recursive, kill_after_timeout=15)
+		else:
+			repo = git.Repo.init(gitDir)
+			origin = repo.create_remote("origin", url=gitUrl)
+			assert origin.exists()
+			origin.fetch()
+			origin.pull(ref)
+
+			for submodule in repo.submodules:
+				submodule.update(init=True)
+
+		return gitDir, repo
+
+	except GitCommandError as e:
+		# This is needed to stop the backtrace being weird
+		err = e.stderr
+
+	except gitdb.exc.BadName as e:
+		err = "Unable to find the reference " + (ref or "?") + "\n" + e.stderr
+
+	raise TaskError(err.replace("stderr: ", "") \
+			.replace("Cloning into '" + gitDir + "'...", "") \
+			.strip())
+
+
+@celery.task(bind=True)
+def updateMetaFromRelease(self, id, path):
+	release = PackageRelease.query.get(id)
+	if release is None:
+		raise TaskError("No such release!")
+	elif release.package is None:
+		raise TaskError("No package attached to release")
+
+	temp = getTempDir()
+	try:
+		with ZipFile(path, 'r') as zip_ref:
+			zip_ref.extractall(temp)
+
+		try:
+			tree = build_tree(temp, expected_type=ContentType[release.package.type.name], \
+				author=release.package.author.username, name=release.package.name)
+
+			cache = {}
+			def getMetaPackages(names):
+				return [ MetaPackage.GetOrCreate(x, cache) for x in names ]
 
-def parseConf(string):
-	retval = {}
-	for line in string.split("\n"):
-		idx = line.find("=")
-		if idx > 0:
-			key   = line[:idx].strip()
-			value = line[idx+1:].strip()
-			retval[key] = value
+			provides = getMetaPackages(tree.getModNames())
 
-	return retval
+			package = release.package
+			package.provides.clear()
+			package.provides.extend(provides)
+
+			for dep in package.dependencies:
+				if dep.meta_package:
+					db.session.delete(dep)
+
+			for meta in getMetaPackages(tree.fold("meta", "depends")):
+				db.session.add(Dependency(package, meta=meta, optional=False))
+
+			for meta in getMetaPackages(tree.fold("meta", "optional_depends")):
+				db.session.add(Dependency(package, meta=meta, optional=True))
+
+			db.session.commit()
+
+		except MinetestCheckError as err:
+			if "Fails validation" not in release.title:
+				release.title += " (Fails validation)"
+
+			release.task_id = self.request.id
+			release.approved = False
+			db.session.commit()
+
+			raise TaskError(str(err))
+
+	finally:
+		shutil.rmtree(temp)
 
 
 @celery.task()
 def getMeta(urlstr, author):
-	url = urlparse(urlstr)
+	gitDir, _ = cloneRepo(urlstr, recursive=True)
 
-	urlmaker = None
-	if url.netloc == "github.com":
-		urlmaker = GithubURLMaker(url)
-	else:
-		raise TaskError("Unsupported repo")
+	try:
+		tree = build_tree(gitDir, author=author, repo=urlstr)
+	except MinetestCheckError as err:
+		raise TaskError(str(err))
 
-	if not urlmaker.isValid():
-		raise TaskError("Error! Url maker not valid")
+	shutil.rmtree(gitDir)
 
 	result = {}
+	result["name"] = tree.name
+	result["provides"] = tree.getModNames()
+	result["type"] = tree.type.name
 
-	result["repo"] = urlmaker.getRepoURL()
-	result["issueTracker"] = urlmaker.getIssueTrackerURL()
+	for key in ["depends", "optional_depends"]:
+		result[key] = tree.fold("meta", key)
+
+	for key in ["title", "repo", "issueTracker", "forumId", "description", "short_description"]:
+		result[key] = tree.get(key)
+
+	for mod in result["provides"]:
+		result["depends"].discard(mod)
+		result["optional_depends"].discard(mod)
+
+	for key, value in result.items():
+		if isinstance(value, set):
+			result[key] = list(value)
+
+	return result
+
+
+@celery.task(bind=True)
+def checkZipRelease(self, id, path):
+	release = PackageRelease.query.get(id)
+	if release is None:
+		raise TaskError("No such release!")
+	elif release.package is None:
+		raise TaskError("No package attached to release")
 
+	temp = getTempDir()
 	try:
-		contents = urllib.request.urlopen(urlmaker.getModConfURL()).read().decode("utf-8")
-		conf = parseConf(contents)
-		for key in ["name", "description", "title"]:
-			try:
-				result[key] = conf[key]
-			except KeyError:
-				pass
-	except HTTPError:
-		print("mod.conf does not exist")
-
-	if "name" in result:
-		result["title"] = result["name"].replace("_", " ").title()
-
-	if not "description" in result:
+		with ZipFile(path, 'r') as zip_ref:
+			zip_ref.extractall(temp)
+
 		try:
-			contents = urllib.request.urlopen(urlmaker.getDescURL()).read().decode("utf-8")
-			result["description"] = contents.strip()
-		except HTTPError:
-			print("description.txt does not exist!")
+			tree = build_tree(temp, expected_type=ContentType[release.package.type.name], \
+				author=release.package.author.username, name=release.package.name)
+		except MinetestCheckError as err:
+			if "Fails validation" not in release.title:
+				release.title += " (Fails validation)"
 
-	if "description" in result:
-		desc = result["description"]
-		idx = desc.find(".") + 1
-		cutIdx = min(len(desc), 200 if idx < 5 else idx)
-		result["short_description"] = desc[:cutIdx]
+			release.task_id = self.request.id
+			release.approved = False
+			db.session.commit()
 
-	info = findModInfo(author, result.get("name"), result["repo"])
-	if info is not None:
-		result["forumId"] = info.get("topicId")
+			raise TaskError(str(err))
 
-	return result
+		release.task_id = None
+		release.approve(release.package.author)
+		db.session.commit()
+
+	finally:
+		shutil.rmtree(temp)
 
 
 @celery.task()
 def makeVCSRelease(id, branch):
 	release = PackageRelease.query.get(id)
-
 	if release is None:
 		raise TaskError("No such release!")
-
-	if release.package is None:
+	elif release.package is None:
 		raise TaskError("No package attached to release")
 
-	url = urlparse(release.package.repo)
+	gitDir, repo = cloneRepo(release.package.repo, ref=branch, recursive=True)
+
+	tree = None
+	try:
+		tree = build_tree(gitDir, expected_type=ContentType[release.package.type.name], \
+			author=release.package.author.username, name=release.package.name)
+	except MinetestCheckError as err:
+		raise TaskError(str(err))
 
-	urlmaker = None
-	if url.netloc == "github.com":
-		urlmaker = GithubURLMaker(url)
-	else:
-		raise TaskError("Unsupported repo")
+	try:
+		filename = randomString(10) + ".zip"
+		destPath = os.path.join(app.config["UPLOAD_DIR"], filename)
 
-	if not urlmaker.isValid():
-		raise TaskError("Invalid github repo URL")
+		assert(not os.path.isfile(destPath))
+		archiver = GitArchiver(force_sub=True, main_repo_abspath=gitDir)
+		archiver.create(destPath)
+		assert(os.path.isfile(destPath))
 
-	commitsURL = urlmaker.getCommitsURL(branch)
-	contents = urllib.request.urlopen(commitsURL).read().decode("utf-8")
-	commits = json.loads(contents)
+		release.url         = "/uploads/" + filename
+		release.task_id     = None
+		release.commit_hash = repo.head.object.hexsha
 
-	if len(commits) == 0 or not "sha" in commits[0]:
-		raise TaskError("No commits found")
+		if tree.meta.get("min_minetest_version"):
+			release.min_rel = MinetestRelease.get(tree.meta["min_minetest_version"], None)
 
-	release.url = urlmaker.getCommitDownload(commits[0]["sha"])
-	print(release.url)
-	release.task_id = None
-	db.session.commit()
+		if tree.meta.get("max_minetest_version"):
+			release.max_rel = MinetestRelease.get(tree.meta["max_minetest_version"], None)
 
-	return release.url
+		release.approve(release.package.author)
+		db.session.commit()
+
+		return release.url
+	finally:
+		shutil.rmtree(gitDir)
 
 
 @celery.task()
@@ -235,32 +311,33 @@ def importRepoScreenshot(id):
 		raise Exception("Unexpected none package")
 
 	# Get URL Maker
-	url = urlparse(package.repo)
-	urlmaker = None
-	if url.netloc == "github.com":
-		urlmaker = GithubURLMaker(url)
-	else:
-		raise TaskError("Unsupported repo")
-
-	if not urlmaker.isValid():
-		raise TaskError("Error! Url maker not valid")
-
 	try:
-		filename = randomString(10) + ".png"
-		imagePath = os.path.join("app/public/uploads", filename)
-		print(imagePath)
-		urllib.request.urlretrieve(urlmaker.getScreenshotURL(), imagePath)
-
-		ss = PackageScreenshot()
-		ss.approved = True
-		ss.package = package
-		ss.title   = "screenshot.png"
-		ss.url     = "/uploads/" + filename
-		db.session.add(ss)
-		db.session.commit()
-
-		return "/uploads/" + filename
-	except HTTPError:
-		print("screenshot.png does not exist")
+		gitDir, _ = cloneRepo(package.repo)
+	except TaskError as e:
+		# ignore download errors
+		print(e)
+		return None
 
+	# Find and import screenshot
+	try:
+		for ext in ["png", "jpg", "jpeg"]:
+			sourcePath = gitDir + "/screenshot." + ext
+			if os.path.isfile(sourcePath):
+				filename = randomString(10) + "." + ext
+				destPath = os.path.join(app.config["UPLOAD_DIR"], filename)
+				shutil.copyfile(sourcePath, destPath)
+
+				ss = PackageScreenshot()
+				ss.approved = True
+				ss.package = package
+				ss.title   = "screenshot.png"
+				ss.url	 = "/uploads/" + filename
+				db.session.add(ss)
+				db.session.commit()
+
+				return "/uploads/" + filename
+	finally:
+		shutil.rmtree(gitDir)
+
+	print("screenshot.png does not exist")
 	return None