2 # Copyright (C) 2018 rubenwardy
4 # This program is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation, either version 3 of the License, or
7 # (at your option) any later version.
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
14 # You should have received a copy of the GNU General Public License
15 # along with this program. If not, see <https://www.gnu.org/licenses/>.
18 import flask, json, os
19 from flask.ext.sqlalchemy import SQLAlchemy
20 from urllib.error import HTTPError
22 from urllib.parse import urlparse, quote_plus
24 from app.models import *
25 from app.tasks import celery, TaskError
26 from app.utils import randomString
29 def __init__(self, url):
32 m = re.search("^\/([^\/]+)\/([^\/]+)\/?$", url.path)
37 repo = m.group(2).replace(".git", "")
38 self.baseUrl = "https://raw.githubusercontent.com/{}/{}/master" \
44 return self.baseUrl is not None
47 return "https://github.com/{}/{}".format(self.user, self.repo)
49 def getIssueTrackerURL(self):
50 return "https://github.com/{}/{}/issues/".format(self.user, self.repo)
52 def getModConfURL(self):
53 return self.baseUrl + "/mod.conf"
56 return self.baseUrl + "/description.txt"
58 def getDependsURL(self):
59 return self.baseUrl + "/depends.txt"
61 def getScreenshotURL(self):
62 return self.baseUrl + "/screenshot.png"
64 def getCommitsURL(self, branch):
65 return "https://api.github.com/repos/{}/{}/commits?sha={}" \
66 .format(self.user, self.repo, urllib.parse.quote_plus(branch))
68 def getCommitDownload(self, commit):
69 return "https://github.com/{}/{}/archive/{}.zip" \
70 .format(self.user, self.repo, commit)
73 krock_list_cache = None
74 krock_list_cache_by_name = None
76 global krock_list_cache
77 global krock_list_cache_by_name
79 if krock_list_cache is None:
80 contents = urllib.request.urlopen("http://krock-works.16mb.com/MTstuff/modList.php").read().decode("utf-8")
81 list = json.loads(contents)
84 if not ("title" in x and "author" in x and \
85 "topicId" in x and "link" in x and x["link"] != ""):
89 m = re.search("\[([A-Za-z0-9_]+)\]", x["title"])
93 x["name"] = m.group(1)
99 "author": x["author"],
101 "topicId": x["topicId"],
105 krock_list_cache = [g(x) for x in list if h(x)]
106 krock_list_cache_by_name = {}
107 for x in krock_list_cache:
108 if not x["name"] in krock_list_cache_by_name:
109 krock_list_cache_by_name[x["name"]] = []
111 krock_list_cache_by_name[x["name"]].append(x)
113 return krock_list_cache, krock_list_cache_by_name
115 def findModInfo(author, name, link):
116 list, lookup = getKrockList()
118 if name is not None and name in lookup:
119 if len(lookup[name]) == 1:
120 return lookup[name][0]
122 for x in lookup[name]:
123 if x["author"] == author:
126 if link is not None and len(link) > 15:
128 if link in x["link"]:
134 def parseConf(string):
136 for line in string.split("\n"):
139 key = line[:idx].strip()
140 value = line[idx+1:].strip()
147 def getMeta(urlstr, author):
148 url = urlparse(urlstr)
151 if url.netloc == "github.com":
152 urlmaker = GithubURLMaker(url)
154 raise TaskError("Unsupported repo")
156 if not urlmaker.isValid():
157 raise TaskError("Error! Url maker not valid")
161 result["repo"] = urlmaker.getRepoURL()
162 result["issueTracker"] = urlmaker.getIssueTrackerURL()
165 contents = urllib.request.urlopen(urlmaker.getModConfURL()).read().decode("utf-8")
166 conf = parseConf(contents)
167 for key in ["name", "description", "title", "depends", "optional_depends"]:
169 result[key] = conf[key]
173 print("mod.conf does not exist")
176 result["title"] = result["name"].replace("_", " ").title()
178 if not "description" in result:
180 contents = urllib.request.urlopen(urlmaker.getDescURL()).read().decode("utf-8")
181 result["description"] = contents.strip()
183 print("description.txt does not exist!")
186 pattern = re.compile("^([a-z0-9_]+)\??$")
187 if not "depends" in result and not "optional_depends" in result:
189 contents = urllib.request.urlopen(urlmaker.getDependsURL()).read().decode("utf-8")
192 for line in contents.split("\n"):
194 if pattern.match(line):
195 if line[len(line) - 1] == "?":
196 soft.append( line[:-1])
200 result["depends"] = ",".join(hard)
201 result["optional_depends"] = ",".join(soft)
205 print("depends.txt does not exist!")
207 if "description" in result:
208 desc = result["description"]
209 idx = desc.find(".") + 1
210 cutIdx = min(len(desc), 200 if idx < 5 else idx)
211 result["short_description"] = desc[:cutIdx]
214 info = findModInfo(author, result.get("name"), result["repo"])
216 result["forumId"] = info.get("topicId")
222 def makeVCSRelease(id, branch):
223 release = PackageRelease.query.get(id)
226 raise TaskError("No such release!")
228 if release.package is None:
229 raise TaskError("No package attached to release")
231 url = urlparse(release.package.repo)
234 if url.netloc == "github.com":
235 urlmaker = GithubURLMaker(url)
237 raise TaskError("Unsupported repo")
239 if not urlmaker.isValid():
240 raise TaskError("Invalid github repo URL")
242 commitsURL = urlmaker.getCommitsURL(branch)
243 contents = urllib.request.urlopen(commitsURL).read().decode("utf-8")
244 commits = json.loads(contents)
246 if len(commits) == 0 or not "sha" in commits[0]:
247 raise TaskError("No commits found")
249 release.url = urlmaker.getCommitDownload(commits[0]["sha"])
251 release.task_id = None
258 def importRepoScreenshot(id):
259 package = Package.query.get(id)
260 if package is None or package.soft_deleted:
261 raise Exception("Unexpected none package")
264 url = urlparse(package.repo)
266 if url.netloc == "github.com":
267 urlmaker = GithubURLMaker(url)
269 raise TaskError("Unsupported repo")
271 if not urlmaker.isValid():
272 raise TaskError("Error! Url maker not valid")
275 filename = randomString(10) + ".png"
276 imagePath = os.path.join("app/public/uploads", filename)
278 urllib.request.urlretrieve(urlmaker.getScreenshotURL(), imagePath)
280 ss = PackageScreenshot()
283 ss.title = "screenshot.png"
284 ss.url = "/uploads/" + filename
288 return "/uploads/" + filename
290 print("screenshot.png does not exist")
296 def getDepends(package):
297 url = urlparse(package.repo)
299 if url.netloc == "github.com":
300 urlmaker = GithubURLMaker(url)
302 raise TaskError("Unsupported repo")
305 if urlmaker.isValid():
307 # Try getting depends on mod.conf
310 contents = urllib.request.urlopen(urlmaker.getModConfURL()).read().decode("utf-8")
311 conf = parseConf(contents)
312 for key in ["depends", "optional_depends"]:
314 result[key] = conf[key]
319 print("mod.conf does not exist")
321 if "depends" in result or "optional_depends" in result:
329 pattern = re.compile("^([a-z0-9_]+)\??$")
331 contents = urllib.request.urlopen(urlmaker.getDependsURL()).read().decode("utf-8")
334 for line in contents.split("\n"):
336 if pattern.match(line):
337 if line[len(line) - 1] == "?":
338 soft.append( line[:-1])
342 result["depends"] = ",".join(hard)
343 result["optional_depends"] = ",".join(soft)
345 print("depends.txt does not exist")
350 print(TaskError("non-github depends detector not implemented yet!"))
354 def importDependencies(package, mpackage_cache):
355 if Dependency.query.filter_by(depender=package).count() != 0:
358 result = getDepends(package)
360 if "depends" in result:
361 deps = Dependency.SpecToList(package, result["depends"], mpackage_cache)
362 print("{} hard: {}".format(len(deps), result["depends"]))
367 if "optional_depends" in result:
368 deps = Dependency.SpecToList(package, result["optional_depends"], mpackage_cache)
369 print("{} soft: {}".format(len(deps), result["optional_depends"]))
375 def importAllDependencies():
376 Dependency.query.delete()
378 packages = Package.query.filter_by(type=PackageType.MOD).all()
379 for i, p in enumerate(packages):
380 print("============= {} ({}/{}) =============".format(p.name, i, len(packages)))
381 importDependencies(p, mpackage_cache)