Add admin function to import foreign release URLs

diff --git a/app/blueprints/admin/admin.py b/app/blueprints/admin/admin.py
index 01bd196b0bf5b25e41eacfe71621ba6a3cbb4546..382dda80093084a5c2ffd3e668abc04913a2732d 100644 (file)
--- a/app/blueprints/admin/admin.py
+++ b/app/blueprints/admin/admin.py
@@ -21,7 +21,7 @@ import flask_menu as menu
 from . import bp
 from app.models import *
 from celery import uuid, group
-from app.tasks.importtasks import importRepoScreenshot, makeVCSRelease, checkZipRelease, updateMetaFromRelease
+from app.tasks.importtasks import importRepoScreenshot, makeVCSRelease, checkZipRelease, updateMetaFromRelease, importForeignDownloads
 from app.tasks.forumtasks  import importTopicList, checkAllForumAccounts
 from flask_wtf import FlaskForm
 from wtforms import *
@@ -71,6 +71,21 @@ def admin_page():
 
                        return redirect(url_for("todo.view"))
 
+               elif action == "importforeign":
+                       releases = PackageRelease.query.filter(PackageRelease.url.like("http%")).all()
+
+                       tasks = []
+                       for release in releases:
+                               tasks.append(importForeignDownloads.s(release.id))
+
+                       result = group(tasks).apply_async()
+
+                       while not result.ready():
+                               import time
+                               time.sleep(0.1)
+
+                       return redirect(url_for("todo.view"))
+
                elif action == "importmodlist":
                        task = importTopicList.delay()
                        return redirect(url_for("tasks.check", id=task.id, r=url_for("todo.topics")))

diff --git a/app/tasks/importtasks.py b/app/tasks/importtasks.py
index 0e5536c54da955ba09c75f418e267f30d8a789dc..e0b2308af89427807ebfb8a2520d451a512d428b 100644 (file)
--- a/app/tasks/importtasks.py
+++ b/app/tasks/importtasks.py
@@ -27,7 +27,7 @@ from zipfile import ZipFile
 from app import app
 from app.models import *
 from app.tasks import celery, TaskError
-from app.utils import randomString
+from app.utils import randomString, getExtension
 from .minetestcheck import build_tree, MinetestCheckError, ContentType
 from .minetestcheck.config import parse_conf
 
@@ -341,3 +341,27 @@ def importRepoScreenshot(id):
 
        print("screenshot.png does not exist")
        return None
+
+
+@celery.task(bind=True)
+def importForeignDownloads(self, id):
+       release = PackageRelease.query.get(id)
+       if release is None:
+               raise TaskError("No such release!")
+       elif release.package is None:
+               raise TaskError("No package attached to release")
+       elif not release.url.startswith("http"):
+               return
+
+       try:
+               ext = getExtension(release.url)
+               filename = randomString(10) + "." + ext
+               filepath = os.path.join(app.config["UPLOAD_DIR"], filename)
+               urllib.request.urlretrieve(release.url, filepath)
+
+               release.url = "/uploads/" + filename
+               db.session.commit()
+       except urllib.error.URLError:
+               release.task_id = self.request.id
+               release.approved = False
+               db.session.commit()

diff --git a/app/templates/admin/list.html b/app/templates/admin/list.html
index faa665244ed374b318fafba02424c053242b4b68..75fb61e6b5680e93184c0d76d3f3ed36c07eb687 100644 (file)
--- a/app/templates/admin/list.html
+++ b/app/templates/admin/list.html
@@ -28,6 +28,7 @@
                                                        <option value="checkreleases">Validate all Zip releases</option>
                                                        <option value="reimportpackages">Reimport meta</option>
                                                        <option value="importmodlist">Import forum topics</option>
+                                                       <option value="importforeign">Import foreign release downloads</option>
                                                        <option value="recalcscores">Recalculate package scores</option>
                                                        <option value="checkusers">Check forum users</option>
                                                        <option value="importscreenshots">Import screenshots from VCS</option>