Improve admin webui (#112)

* fix columns and filters

- add formatters to remove milliseconds on dates
- add column labels to be used with filters
- add new filters for version and build views

* fix screenshot management

- remove edit function
- fix file removal with row deletion

* fix Babel initialisation

* add service view and api check

- add service view to admin interface
- add service check on package upload

* add and maintain md5 hashes

- add a functions to get md5 hash in build model and spk
- use function in build model when uploading build
- use function in spk when signing/unsigning spk

* remove builds from architecture create form

* fix build environment and model

- fix alembic setup environment
- update model to match database
- fix default sort for screenshot

* Update nas test for major_version check

* add major_version check

- identify the major DSM version based on a closest match to the build
- filter package versions based on match to major DSM version
- include earlier "noarch" package version when major DSM  version < 6

* fix download counter

- rewrite catalog download links using md5 hashes for id
- allow downloads of noarch builds to pass arch checks
- rewrite nas tests for new url structure

* Fix depopulate db function

* Revert "fix download counter"

This reverts commit bea7718.

* Amend major_version check

- Add type column to Firmware table
- Increase length of version column
- Filter by type for closest firmware when getting catalog
- Update populate_db with firmware type

* add validators for firmware input

* fix Popped wrong app context.

* fix 500 page on users:
'NoneType' object has no attribute 'strftime'

BaseModelView.index_view() got an unexpected keyword argument 'cls'

Flask-SQLAlchemy uses a custom formatter to handle date formatting when displaying data in the templates. It dosn't support overriding the value.

* fix column formatting

---------

Co-authored-by: publicarray <[email protected]>

.flake8

@@ -1,7 +1,7 @@
 [flake8]
 max-line-length = 88
 extend-ignore = E203
-per-file-ignores = __init__.py:F401 spkrepo/app.py:F841
+per-file-ignores = __init__.py:F401
 exclude =
     docs/*
     migrations/*

migrations/alembic.ini

@@ -1,5 +1,5 @@
 [alembic]
-script_location = .
+script_location = ./migrations
 
 [loggers]
 keys = root,sqlalchemy,alembic

migrations/env.py

@@ -8,7 +8,6 @@
 config = context.config
 fileConfig(config.config_file_name)
 
-
 config.set_main_option(
     "sqlalchemy.url", current_app.config.get("SQLALCHEMY_DATABASE_URI")
 )

migrations/versions/dc7687894ba7_increase_field_sizes.py

@@ -13,7 +13,6 @@
 
 
 def upgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
     op.alter_column(
         "version",
         "conf_dependencies",
@@ -42,11 +41,9 @@ def upgrade():
         type_=sa.UnicodeText(),
         existing_nullable=True,
     )
-    # ### end Alembic commands ###
 
 
 def downgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
     op.alter_column(
         "version",
         "conf_resource",
@@ -75,4 +72,3 @@ def downgrade():
         type_=sa.VARCHAR(length=255),
         existing_nullable=True,
     )
-    # ### end Alembic commands ###

migrations/versions/f95855ce9471_add_firmware_type_and_increase_version_.py

@@ -0,0 +1,47 @@
+"""Add firmware type and increase version length
+
+Revision ID: f95855ce9471
+Revises: 76d559b4e873
+Create Date: 2024-01-15 13:58:34.160242
+
+"""
+revision = "f95855ce9471"
+down_revision = "76d559b4e873"
+
+import sqlalchemy as sa
+from alembic import op
+
+
+def upgrade():
+    op.add_column("firmware", sa.Column("type", sa.Unicode(length=4)))
+    # Set type based on version
+    op.execute(
+        """
+        UPDATE firmware
+        SET type = CASE
+            WHEN version LIKE '1.%' THEN 'srm'
+            ELSE 'dsm'
+        END
+    """
+    )
+    # Modify the column to be NOT NULL after setting the values
+    op.alter_column("firmware", "type", nullable=False)
+
+    op.alter_column(
+        "firmware",
+        "version",
+        existing_type=sa.VARCHAR(length=3),
+        type_=sa.Unicode(length=4),
+        existing_nullable=False,
+    )
+
+
+def downgrade():
+    op.alter_column(
+        "firmware",
+        "version",
+        existing_type=sa.Unicode(length=4),
+        type_=sa.VARCHAR(length=3),
+        existing_nullable=False,
+    )
+    op.drop_column("firmware", "type")

spkrepo/app.py

@@ -2,12 +2,11 @@
 import jinja2
 from flask import Flask
 from flask_admin import Admin
-from flask_babel import Babel
 from wtforms import HiddenField
 
 from . import config as default_config
 from .cli import spkrepo as spkrepo_cli
-from .ext import cache, db, debug_toolbar, mail, migrate, security
+from .ext import babel, cache, db, debug_toolbar, mail, migrate, security
 from .models import user_datastore
 from .views import (
     ArchitectureView,
@@ -16,6 +15,7 @@
     IndexView,
     PackageView,
     ScreenshotView,
+    ServiceView,
     SpkrepoConfirmRegisterForm,
     UserView,
     VersionView,
@@ -53,18 +53,19 @@ def create_app(config=None, register_blueprints=True, init_admin=True):
         admin.add_view(UserView())
         admin.add_view(ArchitectureView())
         admin.add_view(FirmwareView())
+        admin.add_view(ServiceView())
         admin.add_view(ScreenshotView())
         admin.add_view(PackageView())
         admin.add_view(VersionView())
         admin.add_view(BuildView())
         admin.init_app(app)
 
-    # Initialize Flask-Babel
-    babel = Babel(app)
-
     # Commands
     app.cli.add_command(spkrepo_cli)
 
+    # Flask-Babel
+    babel.init_app(app)
+
     # SQLAlchemy
     db.init_app(app)
 

spkrepo/cli.py

@@ -180,8 +180,15 @@ def depopulate_db():
     from spkrepo.models import Package
 
     for package in Package.query.all():
-        shutil.rmtree(os.path.join(current_app.config["DATA_PATH"], package.name))
+        # Delete the package and its associated versions and builds
         db.session.delete(package)
+
+        # Remove the directory associated with the package (if it exists)
+        shutil.rmtree(
+            os.path.join(current_app.config["DATA_PATH"], package.name),
+            ignore_errors=True,
+        )
+
     db.session.commit()
 
 

spkrepo/ext.py

@@ -1,11 +1,15 @@
 # -*- coding: utf-8 -*-
+from flask_babel import Babel
 from flask_caching import Cache
 from flask_debugtoolbar import DebugToolbarExtension
 from flask_mail import Mail
 from flask_migrate import Migrate
 from flask_security import Security
 from flask_sqlalchemy import SQLAlchemy
 
+# Flask-Babel
+babel = Babel()
+
 # Cache
 cache = Cache()
 

spkrepo/models.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+import hashlib
 import io
 import os
 import shutil
@@ -133,8 +134,9 @@ class Firmware(db.Model):
 
     # Columns
     id = db.Column(db.Integer, primary_key=True)
-    version = db.Column(db.Unicode(3), nullable=False)
+    version = db.Column(db.Unicode(4), nullable=False)
     build = db.Column(db.Integer, unique=True, nullable=False)
+    type = db.Column(db.Unicode(4), nullable=False)
 
     @classmethod
     def find(cls, build):
@@ -328,7 +330,7 @@ class Build(db.Model):
     publisher_user_id = db.Column(db.Integer, db.ForeignKey("user.id"))
     checksum = db.Column(db.Unicode(32))
     extract_size = db.Column(db.Integer)
-    path = db.Column(db.Unicode(100))
+    path = db.Column(db.Unicode(2048))
     md5 = db.Column(db.Unicode(32))
     insert_date = db.Column(db.DateTime, default=db.func.now(), nullable=False)
     active = db.Column(db.Boolean(), default=False, nullable=False)
@@ -343,7 +345,11 @@ class Build(db.Model):
     )
     firmware = db.relationship("Firmware", lazy=False)
     publisher = db.relationship("User", foreign_keys=[publisher_user_id])
-    downloads = db.relationship("Download", back_populates="build")
+    downloads = db.relationship(
+        "Download",
+        back_populates="build",
+        cascade="save-update, merge, delete, delete-orphan",
+    )
 
     @classmethod
     def generate_filename(cls, package, version, firmware, architectures):
@@ -360,6 +366,24 @@ def save(self, stream):
         ) as f:
             f.write(stream.read())
 
+    def calculate_md5(self):
+        if not self.path:
+            raise ValueError("Path cannot be empty.")
+
+        file_path = os.path.join(current_app.config["DATA_PATH"], self.path)
+
+        if not os.path.exists(file_path):
+            raise FileNotFoundError(f"File not found at path: {file_path}")
+
+        if self.md5 is None:
+            with io.open(file_path, "rb") as f:
+                md5_hash = hashlib.md5()
+                for chunk in iter(lambda: f.read(4096), b""):
+                    md5_hash.update(chunk)
+                return md5_hash.hexdigest()
+
+        return self.md5
+
     def _after_insert(self):
         assert os.path.exists(os.path.join(current_app.config["DATA_PATH"], self.path))
 

spkrepo/tests/common.py

@@ -281,8 +281,7 @@ def create_spk(self, create, extracted, **kwargs):
         with create_spk(self) as spk_stream:
             self.save(spk_stream)
             if self.md5 is None:
-                spk_stream.seek(0)
-                self.md5 = hashlib.md5(spk_stream.read()).hexdigest()
+                self.md5 = self.calculate_md5()
         spk_stream.close()
 
     @classmethod

spkrepo/tests/test_nas.py

@@ -227,11 +227,11 @@ def test_stable_build_active_stable(self):
             catalog[0], build, data, dict(arch="88f628x", build="1594")
         )
 
-    def test_stable_build_active_stable_5004(self):
+    def test_stable_noarch_build_active_stable_5004(self):
         build = BuildFactory(
             active=True,
             version__report_url=None,
-            architectures=[Architecture.find("88f6281", syno=True)],
+            architectures=[Architecture.find("noarch", syno=True)],
             firmware=Firmware.find(1594),
         )
         db.session.commit()
@@ -247,6 +247,23 @@ def test_stable_build_active_stable_5004(self):
             catalog["packages"][0], build, data, dict(arch="88f628x", build="5004")
         )
 
+    def test_stable_arch_build_active_stable_5004(self):
+        BuildFactory(
+            active=True,
+            version__report_url=None,
+            architectures=[Architecture.find("88f6281", syno=True)],
+            firmware=Firmware.find(1594),
+        )
+        db.session.commit()
+        data = dict(arch="88f6281", build="5004", language="enu")
+        response = self.client.post(url_for("nas.catalog"), data=data)
+        self.assert200(response)
+        self.assertHeader(response, "Content-Type", "application/json")
+        catalog = json.loads(response.data.decode())
+        self.assertIn("packages", catalog)
+        self.assertIn("keyrings", catalog)
+        self.assertEqual(len(catalog["packages"]), 0)
+
     def test_stable_build_active_stable_download_count(self):
         package = PackageFactory()
         build = BuildFactory(

spkrepo/utils.py

@@ -58,6 +58,10 @@ class SPK(object):
     #: Regex for files in conf
     conf_filename_re = re.compile(r"^conf/.+$")
 
+    #: Regex for firmware input
+    firmware_version_re = re.compile(r"^\d+\.\d$")
+    firmware_type_re = re.compile(r"^([a-z]){3,}$")
+
     def __init__(self, stream):
         self.info = {}
         self.icons = {}
@@ -345,6 +349,18 @@ def unsign(self):
         self.stream.truncate()
         self.stream.seek(0)
 
+    def calculate_md5(self):
+        md5_hash = hashlib.md5()
+
+        # Ensure the stream position is at the beginning
+        self.stream.seek(0)
+
+        # Update MD5 hash directly from the stream
+        for chunk in iter(lambda: self.stream.read(4096), b""):
+            md5_hash.update(chunk)
+
+        return md5_hash.hexdigest()
+
     def _generate_signature(self, stream, timestamp_url, gnupghome):  # pragma: no cover
         # generate the signature
         gpg = gnupg.GPG(gnupghome=gnupghome)
@@ -386,7 +402,10 @@ def populate_db():
     )
     db.session.execute(
         Firmware.__table__.insert().values(
-            [{"version": "3.1", "build": 1594}, {"version": "5.0", "build": 4458}]
+            [
+                {"version": "3.1", "build": 1594, "type": "dsm"},
+                {"version": "5.0", "build": 4458, "type": "dsm"},
+            ]
         )
     )
     db.session.execute(

spkrepo/views/__init__.py

@@ -6,6 +6,7 @@
     IndexView,
     PackageView,
     ScreenshotView,
+    ServiceView,
     UserView,
     VersionView,
 )