]> git.lizzy.rs Git - cheatdb.git/blob - app/tasks/importtasks.py
Fix dependencies still being added if in provides
[cheatdb.git] / app / tasks / importtasks.py
1 # ContentDB
2 # Copyright (C) 2018  rubenwardy
3 #
4 # This program is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation, either version 3 of the License, or
7 # (at your option) any later version.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU General Public License
15 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
16
17
18 import flask, json, os, git, tempfile, shutil, gitdb
19 from git import GitCommandError
20 from git_archive_all import GitArchiver
21 from flask_sqlalchemy import SQLAlchemy
22 from urllib.error import HTTPError
23 import urllib.request
24 from urllib.parse import urlparse, quote_plus, urlsplit
25 from zipfile import ZipFile
26
27 from app import app
28 from app.models import *
29 from app.tasks import celery, TaskError
30 from app.utils import randomString, getExtension
31 from .minetestcheck import build_tree, MinetestCheckError, ContentType
32 from .minetestcheck.config import parse_conf
33
34 krock_list_cache = None
35 krock_list_cache_by_name = None
36 def getKrockList():
37         global krock_list_cache
38         global krock_list_cache_by_name
39
40         if krock_list_cache is None:
41                 contents = urllib.request.urlopen("https://krock-works.uk.to/minetest/modList.php").read().decode("utf-8")
42                 list = json.loads(contents)
43
44                 def h(x):
45                         if not ("title"   in x and "author" in x and \
46                                         "topicId" in x and "link"   in x and x["link"] != ""):
47                                 return False
48
49                         import re
50                         m = re.search("\[([A-Za-z0-9_]+)\]", x["title"])
51                         if m is None:
52                                 return False
53
54                         x["name"] = m.group(1)
55                         return True
56
57                 def g(x):
58                         return {
59                                 "title":   x["title"],
60                                 "author":  x["author"],
61                                 "name": x["name"],
62                                 "topicId": x["topicId"],
63                                 "link": x["link"],
64                         }
65
66                 krock_list_cache = [g(x) for x in list if h(x)]
67                 krock_list_cache_by_name = {}
68                 for x in krock_list_cache:
69                         if not x["name"] in krock_list_cache_by_name:
70                                 krock_list_cache_by_name[x["name"]] = []
71
72                         krock_list_cache_by_name[x["name"]].append(x)
73
74         return krock_list_cache, krock_list_cache_by_name
75
76 def findModInfo(author, name, link):
77         list, lookup = getKrockList()
78
79         if name is not None and name in lookup:
80                 if len(lookup[name]) == 1:
81                         return lookup[name][0]
82
83                 for x in lookup[name]:
84                         if x["author"] == author:
85                                 return x
86
87         if link is not None and len(link) > 15:
88                 for x in list:
89                         if link in x["link"]:
90                                 return x
91
92         return None
93
94 def generateGitURL(urlstr):
95         scheme, netloc, path, query, frag = urlsplit(urlstr)
96
97         return "http://:@" + netloc + path + query
98
99
100 def getTempDir():
101         return os.path.join(tempfile.gettempdir(), randomString(10))
102
103
104 # Clones a repo from an unvalidated URL.
105 # Returns a tuple of path and repo on sucess.
106 # Throws `TaskError` on failure.
107 # Caller is responsible for deleting returned directory.
108 def cloneRepo(urlstr, ref=None, recursive=False):
109         gitDir = getTempDir()
110
111         err = None
112         try:
113                 gitUrl = generateGitURL(urlstr)
114                 print("Cloning from " + gitUrl)
115
116                 if ref is None:
117                         repo = git.Repo.clone_from(gitUrl, gitDir, \
118                                         progress=None, env=None, depth=1, recursive=recursive, kill_after_timeout=15)
119                 else:
120                         repo = git.Repo.init(gitDir)
121                         origin = repo.create_remote("origin", url=gitUrl)
122                         assert origin.exists()
123                         origin.fetch()
124                         origin.pull(ref)
125
126                         for submodule in repo.submodules:
127                                 submodule.update(init=True)
128
129                 return gitDir, repo
130
131         except GitCommandError as e:
132                 # This is needed to stop the backtrace being weird
133                 err = e.stderr
134
135         except gitdb.exc.BadName as e:
136                 err = "Unable to find the reference " + (ref or "?") + "\n" + e.stderr
137
138         raise TaskError(err.replace("stderr: ", "") \
139                         .replace("Cloning into '" + gitDir + "'...", "") \
140                         .strip())
141
142
143 @celery.task(bind=True)
144 def updateMetaFromRelease(self, id, path):
145         release = PackageRelease.query.get(id)
146         if release is None:
147                 raise TaskError("No such release!")
148         elif release.package is None:
149                 raise TaskError("No package attached to release")
150
151         temp = getTempDir()
152         try:
153                 with ZipFile(path, 'r') as zip_ref:
154                         zip_ref.extractall(temp)
155
156                 try:
157                         tree = build_tree(temp, expected_type=ContentType[release.package.type.name], \
158                                 author=release.package.author.username, name=release.package.name)
159
160                         cache = {}
161                         def getMetaPackages(names):
162                                 return [ MetaPackage.GetOrCreate(x, cache) for x in names ]
163
164                         provides = getMetaPackages(tree.getModNames())
165
166                         package = release.package
167                         package.provides.clear()
168                         package.provides.extend(provides)
169
170                         # Delete all meta package dependencies
171                         package.dependencies.filter(Dependency.meta_package != None).delete()
172
173                         # Get raw dependencies
174                         depends = tree.fold("meta", "depends")
175                         optional_depends = tree.fold("meta", "optional_depends")
176
177                         # Filter out provides
178                         for mod in provides:
179                                 depends.discard(mod)
180                                 optional_depends.discard(mod)
181
182                         # Add dependencies
183
184                         for meta in getMetaPackages(depends):
185                                 db.session.add(Dependency(package, meta=meta, optional=False))
186
187                         for meta in getMetaPackages(optional_depends):
188                                 db.session.add(Dependency(package, meta=meta, optional=True))
189
190
191                         db.session.commit()
192
193                 except MinetestCheckError as err:
194                         if "Fails validation" not in release.title:
195                                 release.title += " (Fails validation)"
196
197                         release.task_id = self.request.id
198                         release.approved = False
199                         db.session.commit()
200
201                         raise TaskError(str(err))
202
203         finally:
204                 shutil.rmtree(temp)
205
206
207 @celery.task()
208 def getMeta(urlstr, author):
209         gitDir, _ = cloneRepo(urlstr, recursive=True)
210
211         try:
212                 tree = build_tree(gitDir, author=author, repo=urlstr)
213         except MinetestCheckError as err:
214                 raise TaskError(str(err))
215
216         shutil.rmtree(gitDir)
217
218         result = {}
219         result["name"] = tree.name
220         result["provides"] = tree.getModNames()
221         result["type"] = tree.type.name
222
223         for key in ["depends", "optional_depends"]:
224                 result[key] = tree.fold("meta", key)
225
226         for key in ["title", "repo", "issueTracker", "forumId", "description", "short_description"]:
227                 result[key] = tree.get(key)
228
229         for mod in result["provides"]:
230                 result["depends"].discard(mod)
231                 result["optional_depends"].discard(mod)
232
233         for key, value in result.items():
234                 if isinstance(value, set):
235                         result[key] = list(value)
236
237         return result
238
239
240 @celery.task(bind=True)
241 def checkZipRelease(self, id, path):
242         release = PackageRelease.query.get(id)
243         if release is None:
244                 raise TaskError("No such release!")
245         elif release.package is None:
246                 raise TaskError("No package attached to release")
247
248         temp = getTempDir()
249         try:
250                 with ZipFile(path, 'r') as zip_ref:
251                         zip_ref.extractall(temp)
252
253                 try:
254                         tree = build_tree(temp, expected_type=ContentType[release.package.type.name], \
255                                 author=release.package.author.username, name=release.package.name)
256                 except MinetestCheckError as err:
257                         if "Fails validation" not in release.title:
258                                 release.title += " (Fails validation)"
259
260                         release.task_id = self.request.id
261                         release.approved = False
262                         db.session.commit()
263
264                         raise TaskError(str(err))
265
266                 release.task_id = None
267                 release.approve(release.package.author)
268                 db.session.commit()
269
270         finally:
271                 shutil.rmtree(temp)
272
273
274 @celery.task()
275 def makeVCSRelease(id, branch):
276         release = PackageRelease.query.get(id)
277         if release is None:
278                 raise TaskError("No such release!")
279         elif release.package is None:
280                 raise TaskError("No package attached to release")
281
282         gitDir, repo = cloneRepo(release.package.repo, ref=branch, recursive=True)
283
284         tree = None
285         try:
286                 tree = build_tree(gitDir, expected_type=ContentType[release.package.type.name], \
287                         author=release.package.author.username, name=release.package.name)
288         except MinetestCheckError as err:
289                 raise TaskError(str(err))
290
291         try:
292                 filename = randomString(10) + ".zip"
293                 destPath = os.path.join(app.config["UPLOAD_DIR"], filename)
294
295                 assert(not os.path.isfile(destPath))
296                 archiver = GitArchiver(force_sub=True, main_repo_abspath=gitDir)
297                 archiver.create(destPath)
298                 assert(os.path.isfile(destPath))
299
300                 release.url         = "/uploads/" + filename
301                 release.task_id     = None
302                 release.commit_hash = repo.head.object.hexsha
303
304                 if tree.meta.get("min_minetest_version"):
305                         release.min_rel = MinetestRelease.get(tree.meta["min_minetest_version"], None)
306
307                 if tree.meta.get("max_minetest_version"):
308                         release.max_rel = MinetestRelease.get(tree.meta["max_minetest_version"], None)
309
310                 release.approve(release.package.author)
311                 db.session.commit()
312
313                 updateMetaFromRelease.delay(release.id, destPath)
314
315                 return release.url
316         finally:
317                 shutil.rmtree(gitDir)
318
319
320 @celery.task()
321 def importRepoScreenshot(id):
322         package = Package.query.get(id)
323         if package is None or package.soft_deleted:
324                 raise Exception("Unexpected none package")
325
326         # Get URL Maker
327         try:
328                 gitDir, _ = cloneRepo(package.repo)
329         except TaskError as e:
330                 # ignore download errors
331                 print(e)
332                 return None
333
334         # Find and import screenshot
335         try:
336                 for ext in ["png", "jpg", "jpeg"]:
337                         sourcePath = gitDir + "/screenshot." + ext
338                         if os.path.isfile(sourcePath):
339                                 filename = randomString(10) + "." + ext
340                                 destPath = os.path.join(app.config["UPLOAD_DIR"], filename)
341                                 shutil.copyfile(sourcePath, destPath)
342
343                                 ss = PackageScreenshot()
344                                 ss.approved = True
345                                 ss.package = package
346                                 ss.title   = "screenshot.png"
347                                 ss.url   = "/uploads/" + filename
348                                 db.session.add(ss)
349                                 db.session.commit()
350
351                                 return "/uploads/" + filename
352         finally:
353                 shutil.rmtree(gitDir)
354
355         print("screenshot.png does not exist")
356         return None
357
358
359 @celery.task(bind=True)
360 def importForeignDownloads(self, id):
361         release = PackageRelease.query.get(id)
362         if release is None:
363                 raise TaskError("No such release!")
364         elif release.package is None:
365                 raise TaskError("No package attached to release")
366         elif not release.url.startswith("http"):
367                 return
368
369         try:
370                 ext = getExtension(release.url)
371                 filename = randomString(10) + "." + ext
372                 filepath = os.path.join(app.config["UPLOAD_DIR"], filename)
373                 urllib.request.urlretrieve(release.url, filepath)
374
375                 release.url = "/uploads/" + filename
376                 db.session.commit()
377         except urllib.error.URLError:
378                 release.task_id = self.request.id
379                 release.approved = False
380                 db.session.commit()