Add bulk dependency importer from Github

diff --git a/app/tasks/importtasks.py b/app/tasks/importtasks.py
index 022f9b3d858713b51e18afcdeabece8bbbf83184..7ccd36c560e5ab5f171db0e267e3f04b32f1b1e6 100644 (file)
--- a/app/tasks/importtasks.py
+++ b/app/tasks/importtasks.py
@@ -215,7 +215,6 @@ def getMeta(urlstr, author):
        if info is not None:
                result["forumId"] = info.get("topicId")
 
-       print(result)
        return result
 
 
@@ -291,3 +290,94 @@ def importRepoScreenshot(id):
                print("screenshot.png does not exist")
 
        return None
+
+
+
+def getDepends(package):
+       url = urlparse(package.repo)
+       urlmaker = None
+       if url.netloc == "github.com":
+               urlmaker = GithubURLMaker(url)
+       else:
+               raise TaskError("Unsupported repo")
+
+       result = {}
+       if urlmaker.isValid():
+               #
+               # Try getting depends on mod.conf
+               #
+               try:
+                       contents = urllib.request.urlopen(urlmaker.getModConfURL()).read().decode("utf-8")
+                       conf = parseConf(contents)
+                       for key in ["depends", "optional_depends"]:
+                               try:
+                                       result[key] = conf[key]
+                               except KeyError:
+                                       pass
+
+               except HTTPError:
+                       print("mod.conf does not exist")
+
+               if "depends" in result or "optional_depends" in result:
+                       return result
+
+
+               #
+               # Try depends.txt
+               #
+               import re
+               pattern = re.compile("^([a-z0-9_]+)\??$")
+               try:
+                       contents = urllib.request.urlopen(urlmaker.getDependsURL()).read().decode("utf-8")
+                       soft = []
+                       hard = []
+                       for line in contents.split("\n"):
+                               line = line.strip()
+                               if pattern.match(line):
+                                       if line[len(line) - 1] == "?":
+                                               soft.append( line[:-1])
+                                       else:
+                                               hard.append(line)
+
+                       result["depends"] = ",".join(hard)
+                       result["optional_depends"] = ",".join(soft)
+               except HTTPError:
+                       print("depends.txt does not exist")
+
+               return result
+
+       else:
+               print(TaskError("non-github depends detector not implemented yet!"))
+               return {}
+
+
+def importDependencies(package, mpackage_cache):
+       if Dependency.query.filter_by(depender=package).count() != 0:
+               return
+
+       result = getDepends(package)
+
+       if "depends" in result:
+               deps = Dependency.SpecToList(package, result["depends"], mpackage_cache)
+               print("{} hard: {}".format(len(deps), result["depends"]))
+               for dep in deps:
+                       dep.optional = False
+                       db.session.add(dep)
+
+       if "optional_depends" in result:
+               deps = Dependency.SpecToList(package, result["optional_depends"], mpackage_cache)
+               print("{} soft: {}".format(len(deps), result["optional_depends"]))
+               for dep in deps:
+                       dep.optional = True
+                       db.session.add(dep)
+
+@celery.task()
+def importAllDependencies():
+       Dependency.query.delete()
+       mpackage_cache = {}
+       packages = Package.query.filter_by(type=PackageType.MOD).all()
+       for i, p in enumerate(packages):
+               print("============= {} ({}/{}) =============".format(p.name, i, len(packages)))
+               importDependencies(p, mpackage_cache)
+
+       db.session.commit()

diff --git a/app/templates/admin/list.html b/app/templates/admin/list.html
index 3c15fa99912416848a780927d074c1b1a06694bf..284919dc6cb338429a6c30f632524fad99219f73 100644 (file)
--- a/app/templates/admin/list.html
+++ b/app/templates/admin/list.html
@@ -17,8 +17,9 @@
                <form method="post" action="" class="box-body">
                        <input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
                        <select name="action">
-                               <option value="importusers" selected>Create users from mod list</option>
+                               <option value="importusers">Create users from mod list</option>
                                <option value="importscreenshots">Import screenshots from VCS</option>
+                               <option value="importdepends" selected>Import dependencies from downloads</option>
                        </select>
                        <input type="submit" value="Perform" />
                </form>

diff --git a/app/views/admin.py b/app/views/admin.py
index 2320cc60da3af71a1387985fc3699391dbb61a36..b1cfed668b8352ec15c1c246910fa2c8fcb4b9da 100644 (file)
--- a/app/views/admin.py
+++ b/app/views/admin.py
@@ -20,7 +20,7 @@ from flask_user import *
 from flask.ext import menu
 from app import app
 from app.models import *
-from app.tasks.importtasks import importRepoScreenshot
+from app.tasks.importtasks import importRepoScreenshot, importAllDependencies
 from app.tasks.forumtasks  import importUsersFromModList
 from flask_wtf import FlaskForm
 from wtforms import *
@@ -52,6 +52,9 @@ def admin_page():
                                package.soft_deleted = False
                                db.session.commit()
                                return redirect(url_for("admin_page"))
+               elif action == "importdepends":
+                       task = importAllDependencies.delay()
+                       return redirect(url_for("check_task", id=task.id, r=url_for("admin_page")))
                else:
                        flash("Unknown action: " + action, "error")