Optimise SQL queries

diff --git a/app/blueprints/api/endpoints.py b/app/blueprints/api/endpoints.py
index 7c7a44a79bef47fa348994de0c604302efab01f2..65af3b0855f4e4c661fa86b56a24fd0559aa5eb7 100644 (file)
--- a/app/blueprints/api/endpoints.py
+++ b/app/blueprints/api/endpoints.py
@@ -28,6 +28,9 @@ from app.querybuilder import QueryBuilder
 
 @bp.route("/api/packages/")
 def packages():
+       import sys
+       print("\n\n############", file=sys.stderr)
+
        qb    = QueryBuilder(request.args)
        query = qb.buildPackageQuery()
        ver   = qb.getMinetestVersion()

diff --git a/app/blueprints/homepage/__init__.py b/app/blueprints/homepage/__init__.py
index df638df77d127bba35db8a14f37bcfd3a42a4b38..5df612f738bb175dae20111b666a5573db39f737 100644 (file)
--- a/app/blueprints/homepage/__init__.py
+++ b/app/blueprints/homepage/__init__.py
@@ -4,17 +4,23 @@ bp = Blueprint("homepage", __name__)
 
 from app.models import *
 import flask_menu as menu
+from sqlalchemy.orm import joinedload
 from sqlalchemy.sql.expression import func
 
 @bp.route("/")
 @menu.register_menu(bp, ".", "Home")
 def home():
+       def join(query):
+               return query.options( \
+                       joinedload(Package.license), \
+                       joinedload(Package.media_license))
+
        query   = Package.query.filter_by(approved=True, soft_deleted=False)
        count   = query.count()
-       new     = query.order_by(db.desc(Package.created_at)).limit(8).all()
-       pop_mod = query.filter_by(type=PackageType.MOD).order_by(db.desc(Package.score)).limit(8).all()
-       pop_gam = query.filter_by(type=PackageType.GAME).order_by(db.desc(Package.score)).limit(4).all()
-       pop_txp = query.filter_by(type=PackageType.TXP).order_by(db.desc(Package.score)).limit(4).all()
+       new     = join(query.order_by(db.desc(Package.created_at))).limit(8).all()
+       pop_mod = join(query.filter_by(type=PackageType.MOD).order_by(db.desc(Package.score))).limit(8).all()
+       pop_gam = join(query.filter_by(type=PackageType.GAME).order_by(db.desc(Package.score))).limit(4).all()
+       pop_txp = join(query.filter_by(type=PackageType.TXP).order_by(db.desc(Package.score))).limit(4).all()
        downloads_result = db.session.query(func.sum(PackageRelease.downloads)).one_or_none()
        downloads = 0 if not downloads_result or not downloads_result[0] else downloads_result[0]
        return render_template("index.html", count=count, downloads=downloads, \

diff --git a/app/blueprints/packages/packages.py b/app/blueprints/packages/packages.py
index a7d5793c1ecbbe3c06656726e2efd707bce98080..700fc44caa46e3be8f52942676b6d3f767bba677 100644 (file)
--- a/app/blueprints/packages/packages.py
+++ b/app/blueprints/packages/packages.py
@@ -31,6 +31,7 @@ from wtforms import *
 from wtforms.validators import *
 from wtforms.ext.sqlalchemy.fields import QuerySelectField, QuerySelectMultipleField
 from sqlalchemy import or_, func
+from sqlalchemy.orm import joinedload, subqueryload
 
 
 @menu.register_menu(bp, ".mods", "Mods", order=11, endpoint_arguments_constructor=lambda: { 'type': 'mod' })
@@ -43,6 +44,11 @@ def list_all():
        query = qb.buildPackageQuery()
        title = qb.title
 
+       query = query.options( \
+                       joinedload(Package.license), \
+                       joinedload(Package.media_license), \
+                       subqueryload(Package.tags))
+
        if qb.lucky:
                package = query.first()
                if package:

diff --git a/app/models.py b/app/models.py
index 4eed0c22b2b27cb66632e205fcb56f212c5a8246..582418943d92c7988a51250f0846931a01198d35 100644 (file)
--- a/app/models.py
+++ b/app/models.py
@@ -147,7 +147,7 @@ class User(db.Model, UserMixin):
        notifications = db.relationship("Notification", primaryjoin="User.id==Notification.user_id")
 
        # causednotifs  = db.relationship("Notification", backref="causer", lazy="dynamic")
-       packages      = db.relationship("Package", backref="author", lazy="dynamic")
+       packages      = db.relationship("Package", backref=db.backref("author", lazy="joined"), lazy="dynamic")
        requests      = db.relationship("EditRequest", backref="author", lazy="dynamic")
        threads       = db.relationship("Thread", backref="author", lazy="dynamic")
        tokens        = db.relationship("APIToken", backref="owner", lazy="dynamic")
@@ -437,12 +437,12 @@ class Package(db.Model):
        forums       = db.Column(db.Integer,     nullable=True)
 
        provides = db.relationship("MetaPackage", \
-                       secondary=provides, lazy="subquery", order_by=db.asc("name"), \
+                       secondary=provides, lazy="select", order_by=db.asc("name"), \
                        backref=db.backref("packages", lazy="dynamic", order_by=db.desc("score")))
 
        dependencies = db.relationship("Dependency", backref="depender", lazy="dynamic", foreign_keys=[Dependency.depender_id])
 
-       tags = db.relationship("Tag", secondary=tags, lazy="subquery",
+       tags = db.relationship("Tag", secondary=tags, lazy="select",
                        backref=db.backref("packages", lazy=True))
 
        releases = db.relationship("PackageRelease", backref="package",
@@ -1135,3 +1135,8 @@ class ForumTopic(db.Model):
 
 # Setup Flask-User
 user_manager = UserManager(app, db, User)
+
+if app.config.get("LOG_SQL"):
+       import logging
+       logging.basicConfig()
+       logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)

diff --git a/config.example.cfg b/config.example.cfg
index 7f2a2981da0f5eb2663aaa8d13e6ae0e66fc8d96..25fe9e895ff34dd13b8b462cc85468a08d3ad7de 100644 (file)
--- a/config.example.cfg
+++ b/config.example.cfg
@@ -32,6 +32,9 @@ MAIL_UTILS_ERROR_SEND_TO = [""]
 UPLOAD_DIR = "/var/cdb/uploads/"
 THUMBNAIL_DIR = "/var/cdb/thumbnails/"
 
+TEMPLATES_AUTO_RELOAD = False
+LOG_SQL = False
+
 LANGUAGES = {
     'en': 'English',
 }