From bf02c0e21f53d891ea3560a3336e46393fecf711 Mon Sep 17 00:00:00 2001 From: Ferdinand Thiessen Date: Sun, 19 Dec 2021 22:20:34 +0100 Subject: [PATCH 01/27] fix(db): Add __repr__ to custom column types, same as done by SQLAlchemy --- flaschengeist/models/__init__.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/flaschengeist/models/__init__.py b/flaschengeist/models/__init__.py index 8cf3850..369bb30 100644 --- a/flaschengeist/models/__init__.py +++ b/flaschengeist/models/__init__.py @@ -1,7 +1,7 @@ import sys import datetime -from sqlalchemy import BigInteger +from sqlalchemy import BigInteger, util from sqlalchemy.dialects import mysql, sqlite from sqlalchemy.types import DateTime, TypeDecorator @@ -50,6 +50,10 @@ class Serial(TypeDecorator): cache_ok = True impl = BigInteger().with_variant(mysql.BIGINT(unsigned=True), "mysql").with_variant(sqlite.INTEGER(), "sqlite") + # https://alembic.sqlalchemy.org/en/latest/autogenerate.html?highlight=custom%20column#affecting-the-rendering-of-types-themselves + def __repr__(self) -> str: + return util.generic_repr(self) + class UtcDateTime(TypeDecorator): """Almost equivalent to `sqlalchemy.types.DateTime` with @@ -85,3 +89,7 @@ class UtcDateTime(TypeDecorator): value = value.astimezone(datetime.timezone.utc) value = value.replace(tzinfo=datetime.timezone.utc) return value + + # https://alembic.sqlalchemy.org/en/latest/autogenerate.html?highlight=custom%20column#affecting-the-rendering-of-types-themselves + def __repr__(self) -> str: + return util.generic_repr(self) From 6a35137a27382a96d2fa0ff8f362f651241f69ef Mon Sep 17 00:00:00 2001 From: Ferdinand Thiessen Date: Sun, 19 Dec 2021 22:11:57 +0100 Subject: [PATCH 02/27] feat(db): Add database migration support, implements #19 Migrations allow us to keep track of database changes and upgrading databases if needed. * Add initial migrations for core Flaschengeist * Add migrations to balance * Add migrations to pricelist * Skip plugins with not satisfied dependencies. --- flaschengeist/alembic/__init__.py | 0 flaschengeist/alembic/alembic.ini | 53 +++++++ flaschengeist/alembic/env.py | 74 +++++++++ .../255b93b6beed_flaschengeist_initial.py | 132 ++++++++++++++++ flaschengeist/alembic/migrations/__init__.py | 0 flaschengeist/alembic/script.py.mako | 25 ++++ flaschengeist/app.py | 33 +--- flaschengeist/database.py | 44 ++++++ flaschengeist/plugins/__init__.py | 36 ++++- flaschengeist/plugins/auth/__init__.py | 4 +- flaschengeist/plugins/auth_ldap/__init__.py | 2 +- flaschengeist/plugins/auth_plain/__init__.py | 2 + flaschengeist/plugins/balance/__init__.py | 6 +- .../98f2733bbe45_balance_initial.py | 47 ++++++ flaschengeist/plugins/message_mail.py | 2 +- flaschengeist/plugins/pricelist/__init__.py | 2 + .../58ab9b6a8839_pricelist_initial.py | 141 ++++++++++++++++++ setup.cfg | 8 +- 18 files changed, 571 insertions(+), 40 deletions(-) create mode 100644 flaschengeist/alembic/__init__.py create mode 100644 flaschengeist/alembic/alembic.ini create mode 100644 flaschengeist/alembic/env.py create mode 100644 flaschengeist/alembic/migrations/255b93b6beed_flaschengeist_initial.py create mode 100644 flaschengeist/alembic/migrations/__init__.py create mode 100644 flaschengeist/alembic/script.py.mako create mode 100644 flaschengeist/plugins/balance/migrations/98f2733bbe45_balance_initial.py create mode 100644 flaschengeist/plugins/pricelist/migrations/58ab9b6a8839_pricelist_initial.py diff --git a/flaschengeist/alembic/__init__.py b/flaschengeist/alembic/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/flaschengeist/alembic/alembic.ini b/flaschengeist/alembic/alembic.ini new file mode 100644 index 0000000..f9e9d8e --- /dev/null +++ b/flaschengeist/alembic/alembic.ini @@ -0,0 +1,53 @@ +# A generic, single database configuration. +# No used by flaschengeist + +[alembic] +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +version_path_separator = os +version_locations = %(here)s/migrations + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic,flask_migrate + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[logger_flask_migrate] +level = INFO +handlers = +qualname = flask_migrate + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/flaschengeist/alembic/env.py b/flaschengeist/alembic/env.py new file mode 100644 index 0000000..f8e05d5 --- /dev/null +++ b/flaschengeist/alembic/env.py @@ -0,0 +1,74 @@ +import logging +from logging.config import fileConfig +from pathlib import Path +from flask import current_app +from alembic import context + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(Path(config.get_main_option("script_location")) / config.config_file_name.split("/")[-1]) +logger = logging.getLogger("alembic.env") + +config.set_main_option("sqlalchemy.url", str(current_app.extensions["migrate"].db.get_engine().url).replace("%", "%%")) +target_metadata = current_app.extensions["migrate"].db.metadata + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure(url=url, target_metadata=target_metadata, literal_binds=True) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + + # this callback is used to prevent an auto-migration from being generated + # when there are no changes to the schema + # reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html + def process_revision_directives(context, revision, directives): + if getattr(config.cmd_opts, "autogenerate", False): + script = directives[0] + if script.upgrade_ops.is_empty(): + directives[:] = [] + logger.info("No changes in schema detected.") + + connectable = current_app.extensions["migrate"].db.get_engine() + + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata, + process_revision_directives=process_revision_directives, + **current_app.extensions["migrate"].configure_args, + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/flaschengeist/alembic/migrations/255b93b6beed_flaschengeist_initial.py b/flaschengeist/alembic/migrations/255b93b6beed_flaschengeist_initial.py new file mode 100644 index 0000000..b7deac6 --- /dev/null +++ b/flaschengeist/alembic/migrations/255b93b6beed_flaschengeist_initial.py @@ -0,0 +1,132 @@ +"""Flaschengeist: Initial + +Revision ID: 255b93b6beed +Revises: +Create Date: 2022-02-23 14:33:02.851388 + +""" +from alembic import op +import sqlalchemy as sa +import flaschengeist + + +# revision identifiers, used by Alembic. +revision = "255b93b6beed" +down_revision = None +branch_labels = ("flaschengeist",) +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "image", + sa.Column("id", flaschengeist.models.Serial(), nullable=False), + sa.Column("filename_", sa.String(length=127), nullable=False), + sa.Column("mimetype_", sa.String(length=30), nullable=False), + sa.Column("thumbnail_", sa.String(length=127), nullable=True), + sa.Column("path_", sa.String(length=127), nullable=True), + sa.PrimaryKeyConstraint("id", name=op.f("pk_image")), + ) + op.create_table( + "permission", + sa.Column("name", sa.String(length=30), nullable=True), + sa.Column("id", flaschengeist.models.Serial(), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_permission")), + sa.UniqueConstraint("name", name=op.f("uq_permission_name")), + ) + op.create_table( + "role", + sa.Column("id", flaschengeist.models.Serial(), nullable=False), + sa.Column("name", sa.String(length=30), nullable=True), + sa.PrimaryKeyConstraint("id", name=op.f("pk_role")), + sa.UniqueConstraint("name", name=op.f("uq_role_name")), + ) + op.create_table( + "role_x_permission", + sa.Column("role_id", flaschengeist.models.Serial(), nullable=True), + sa.Column("permission_id", flaschengeist.models.Serial(), nullable=True), + sa.ForeignKeyConstraint( + ["permission_id"], ["permission.id"], name=op.f("fk_role_x_permission_permission_id_permission") + ), + sa.ForeignKeyConstraint(["role_id"], ["role.id"], name=op.f("fk_role_x_permission_role_id_role")), + ) + op.create_table( + "user", + sa.Column("userid", sa.String(length=30), nullable=False), + sa.Column("display_name", sa.String(length=30), nullable=True), + sa.Column("firstname", sa.String(length=50), nullable=False), + sa.Column("lastname", sa.String(length=50), nullable=False), + sa.Column("deleted", sa.Boolean(), nullable=True), + sa.Column("birthday", sa.Date(), nullable=True), + sa.Column("mail", sa.String(length=60), nullable=True), + sa.Column("id", flaschengeist.models.Serial(), nullable=False), + sa.Column("avatar", flaschengeist.models.Serial(), nullable=True), + sa.ForeignKeyConstraint(["avatar"], ["image.id"], name=op.f("fk_user_avatar_image")), + sa.PrimaryKeyConstraint("id", name=op.f("pk_user")), + sa.UniqueConstraint("userid", name=op.f("uq_user_userid")), + ) + op.create_table( + "notification", + sa.Column("id", flaschengeist.models.Serial(), nullable=False), + sa.Column("plugin", sa.String(length=127), nullable=False), + sa.Column("text", sa.Text(), nullable=True), + sa.Column("data", sa.PickleType(), nullable=True), + sa.Column("time", flaschengeist.models.UtcDateTime(), nullable=False), + sa.Column("user_id", flaschengeist.models.Serial(), nullable=False), + sa.ForeignKeyConstraint(["user_id"], ["user.id"], name=op.f("fk_notification_user_id_user")), + sa.PrimaryKeyConstraint("id", name=op.f("pk_notification")), + ) + op.create_table( + "password_reset", + sa.Column("user", flaschengeist.models.Serial(), nullable=False), + sa.Column("token", sa.String(length=32), nullable=True), + sa.Column("expires", flaschengeist.models.UtcDateTime(), nullable=True), + sa.ForeignKeyConstraint(["user"], ["user.id"], name=op.f("fk_password_reset_user_user")), + sa.PrimaryKeyConstraint("user", name=op.f("pk_password_reset")), + ) + op.create_table( + "session", + sa.Column("expires", flaschengeist.models.UtcDateTime(), nullable=True), + sa.Column("token", sa.String(length=32), nullable=True), + sa.Column("lifetime", sa.Integer(), nullable=True), + sa.Column("browser", sa.String(length=30), nullable=True), + sa.Column("platform", sa.String(length=30), nullable=True), + sa.Column("id", flaschengeist.models.Serial(), nullable=False), + sa.Column("user_id", flaschengeist.models.Serial(), nullable=True), + sa.ForeignKeyConstraint(["user_id"], ["user.id"], name=op.f("fk_session_user_id_user")), + sa.PrimaryKeyConstraint("id", name=op.f("pk_session")), + sa.UniqueConstraint("token", name=op.f("uq_session_token")), + ) + op.create_table( + "user_attribute", + sa.Column("id", flaschengeist.models.Serial(), nullable=False), + sa.Column("user", flaschengeist.models.Serial(), nullable=False), + sa.Column("name", sa.String(length=30), nullable=True), + sa.Column("value", sa.PickleType(), nullable=True), + sa.ForeignKeyConstraint(["user"], ["user.id"], name=op.f("fk_user_attribute_user_user")), + sa.PrimaryKeyConstraint("id", name=op.f("pk_user_attribute")), + ) + op.create_table( + "user_x_role", + sa.Column("user_id", flaschengeist.models.Serial(), nullable=True), + sa.Column("role_id", flaschengeist.models.Serial(), nullable=True), + sa.ForeignKeyConstraint(["role_id"], ["role.id"], name=op.f("fk_user_x_role_role_id_role")), + sa.ForeignKeyConstraint(["user_id"], ["user.id"], name=op.f("fk_user_x_role_user_id_user")), + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("user_x_role") + op.drop_table("user_attribute") + op.drop_table("session") + op.drop_table("password_reset") + op.drop_table("notification") + op.drop_table("user") + op.drop_table("role_x_permission") + op.drop_table("role") + op.drop_table("permission") + op.drop_table("image") + # ### end Alembic commands ### diff --git a/flaschengeist/alembic/migrations/__init__.py b/flaschengeist/alembic/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/flaschengeist/alembic/script.py.mako b/flaschengeist/alembic/script.py.mako new file mode 100644 index 0000000..a6f4fdf --- /dev/null +++ b/flaschengeist/alembic/script.py.mako @@ -0,0 +1,25 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +import flaschengeist +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/flaschengeist/app.py b/flaschengeist/app.py index f2f1664..b6f87e1 100644 --- a/flaschengeist/app.py +++ b/flaschengeist/app.py @@ -1,6 +1,6 @@ import enum -from flask import Flask, current_app +from flask import Flask from flask_cors import CORS from datetime import datetime, date from flask.json import JSONEncoder, jsonify @@ -11,7 +11,6 @@ from werkzeug.exceptions import HTTPException from flaschengeist import logger from flaschengeist.utils.hook import Hook from flaschengeist.plugins import AuthPlugin -from flaschengeist.controller import roleController from flaschengeist.config import config, configure_app @@ -37,10 +36,9 @@ class CustomJSONEncoder(JSONEncoder): @Hook("plugins.loaded") -def __load_plugins(app): - logger.debug("Search for plugins") - +def load_plugins(app: Flask): app.config["FG_PLUGINS"] = {} + for entry_point in entry_points(group="flaschengeist.plugins"): logger.debug(f"Found plugin: {entry_point.name} ({entry_point.dist.version})") @@ -72,37 +70,22 @@ def __load_plugins(app): else: logger.debug(f"Skip disabled plugin {entry_point.name}") if "FG_AUTH_BACKEND" not in app.config: - logger.error("No authentication plugin configured or authentication plugin not found") + logger.fatal("No authentication plugin configured or authentication plugin not found") raise RuntimeError("No authentication plugin configured or authentication plugin not found") -@Hook("plugins.installed") -def install_all(): - from flaschengeist.database import db - - db.create_all() - db.session.commit() - for name, plugin in current_app.config["FG_PLUGINS"].items(): - if not plugin: - logger.debug(f"Skip disabled plugin: {name}") - continue - logger.info(f"Install plugin {name}") - plugin.install() - if plugin.permissions: - roleController.create_permissions(plugin.permissions) - - -def create_app(test_config=None): +def create_app(test_config=None, cli=False): app = Flask("flaschengeist") app.json_encoder = CustomJSONEncoder CORS(app) with app.app_context(): - from flaschengeist.database import db + from flaschengeist.database import db, migrate configure_app(app, test_config) db.init_app(app) - __load_plugins(app) + migrate.init_app(app, db, compare_type=True) + load_plugins(app) @app.route("/", methods=["GET"]) def __get_state(): diff --git a/flaschengeist/database.py b/flaschengeist/database.py index ebda993..a2c4672 100644 --- a/flaschengeist/database.py +++ b/flaschengeist/database.py @@ -1,6 +1,11 @@ +import os +from flask_migrate import Migrate, Config from flask_sqlalchemy import SQLAlchemy +from importlib_metadata import EntryPoint from sqlalchemy import MetaData +from flaschengeist import logger + # https://alembic.sqlalchemy.org/en/latest/naming.html metadata = MetaData( naming_convention={ @@ -14,6 +19,45 @@ metadata = MetaData( db = SQLAlchemy(metadata=metadata) +migrate = Migrate() + + +@migrate.configure +def configure_alembic(config: Config): + """Alembic configuration hook + + Inject all migrations paths into the ``version_locations`` config option. + This includes even disabled plugins, as simply disabling a plugin without + uninstall can break the alembic version management. + """ + from importlib_metadata import entry_points, distribution + + # Set main script location + config.set_main_option( + "script_location", str(distribution("flaschengeist").locate_file("") / "flaschengeist" / "alembic") + ) + + # Set Flaschengeist's migrations + migrations = [config.get_main_option("script_location") + "/migrations"] + + # Gather all migration paths + ep: EntryPoint + for ep in entry_points(group="flaschengeist.plugins"): + try: + directory = ep.dist.locate_file("") + for loc in ep.module.split(".") + ["migrations"]: + directory /= loc + if directory.exists(): + logger.debug(f"Adding migration version path {directory}") + migrations.append(str(directory.resolve())) + except: + logger.warning(f"Could not load migrations of plugin {ep.name} for database migration.") + logger.debug("Plugin loading failed", exc_info=True) + + # write back seperator (we changed it if neither seperator nor locations were specified) + config.set_main_option("version_path_separator", os.pathsep) + config.set_main_option("version_locations", os.pathsep.join(set(migrations))) + return config def case_sensitive(s): diff --git a/flaschengeist/plugins/__init__.py b/flaschengeist/plugins/__init__.py index 87ef13c..49e01e1 100644 --- a/flaschengeist/plugins/__init__.py +++ b/flaschengeist/plugins/__init__.py @@ -3,9 +3,11 @@ .. include:: docs/plugin_development.md """ + +from typing import Optional from importlib_metadata import Distribution, EntryPoint -from werkzeug.datastructures import FileStorage from werkzeug.exceptions import MethodNotAllowed, NotFound +from werkzeug.datastructures import FileStorage from flaschengeist.models.user import _Avatar, User from flaschengeist.utils.hook import HookBefore, HookAfter @@ -77,8 +79,16 @@ class Plugin: models = None """Optional module containing the SQLAlchemy models used by the plugin""" - migrations_path = None - """Optional location of the path to migration files, required if custome db tables are used""" + migrations: Optional[tuple[str, str]] = None + """Optional identifiers of the migration versions + + If custom database tables are used migrations must be provided and the + head and removal versions had to be defined, e.g. + + ``` + migrations = ("head_hash", "removal_hash") + ``` + """ def __init__(self, entry_point: EntryPoint, config=None): """Constructor called by create_app @@ -96,6 +106,17 @@ class Plugin: """ pass + def uninstall(self): + """Uninstall routine + + If the plugin has custom database tables, make sure to remove them. + This can be either done by downgrading the plugin *head* to the *base*. + Or use custom migrations for the uninstall and *stamp* some version. + + Is always called with Flask application context. + """ + pass + def get_setting(self, name: str, **kwargs): """Get plugin setting from database @@ -148,6 +169,11 @@ class Plugin: class AuthPlugin(Plugin): + """Base class for all authentification plugins + + See also `Plugin` + """ + def login(self, user, pw): """Login routine, MUST BE IMPLEMENTED! @@ -210,7 +236,7 @@ class AuthPlugin(Plugin): """ raise MethodNotAllowed - def get_avatar(self, user: User) -> _Avatar: + def get_avatar(self, user): """Retrieve avatar for given user (if supported by auth backend) Default behavior is to use native Image objects, @@ -242,7 +268,7 @@ class AuthPlugin(Plugin): user.avatar_ = imageController.upload_image(file) - def delete_avatar(self, user: User): + def delete_avatar(self, user): """Delete the avatar for given user (if supported by auth backend) Default behavior is to use the imageController and native Image objects. diff --git a/flaschengeist/plugins/auth/__init__.py b/flaschengeist/plugins/auth/__init__.py index 9c08463..66d77be 100644 --- a/flaschengeist/plugins/auth/__init__.py +++ b/flaschengeist/plugins/auth/__init__.py @@ -13,8 +13,8 @@ from flaschengeist.controller import sessionController, userController class AuthRoutePlugin(Plugin): - name = "auth" - blueprint = Blueprint(name, __name__) + id = "dev.flaschengeist.auth" + blueprint = Blueprint("auth", __name__) @AuthRoutePlugin.blueprint.route("/auth", methods=["POST"]) diff --git a/flaschengeist/plugins/auth_ldap/__init__.py b/flaschengeist/plugins/auth_ldap/__init__.py index 7aa8fb6..ef8ecb1 100644 --- a/flaschengeist/plugins/auth_ldap/__init__.py +++ b/flaschengeist/plugins/auth_ldap/__init__.py @@ -18,7 +18,7 @@ from flaschengeist.plugins import AuthPlugin, before_role_updated class AuthLDAP(AuthPlugin): def __init__(self, entry_point, config): - super().__init__(entry_point) + super().__init__(entry_point, config) app.config.update( LDAP_SERVER=config.get("host", "localhost"), LDAP_PORT=config.get("port", 389), diff --git a/flaschengeist/plugins/auth_plain/__init__.py b/flaschengeist/plugins/auth_plain/__init__.py index 10d72d3..e73b98c 100644 --- a/flaschengeist/plugins/auth_plain/__init__.py +++ b/flaschengeist/plugins/auth_plain/__init__.py @@ -14,6 +14,8 @@ from flaschengeist import logger class AuthPlain(AuthPlugin): + id = "auth_plain" + def install(self): plugins_installed(self.post_install) diff --git a/flaschengeist/plugins/balance/__init__.py b/flaschengeist/plugins/balance/__init__.py index c430398..f983a9f 100644 --- a/flaschengeist/plugins/balance/__init__.py +++ b/flaschengeist/plugins/balance/__init__.py @@ -3,7 +3,7 @@ Extends users plugin with balance functions """ -from flask import Blueprint, current_app +from flask import current_app from werkzeug.local import LocalProxy from werkzeug.exceptions import NotFound @@ -57,8 +57,10 @@ def service_debit(): class BalancePlugin(Plugin): permissions = permissions.permissions - plugin: "BalancePlugin" = LocalProxy(lambda: current_app.config["FG_PLUGINS"][BalancePlugin.name]) models = models + migrations = True + + plugin: "BalancePlugin" = LocalProxy(lambda: current_app.config["FG_PLUGINS"][BalancePlugin.name]) def __init__(self, entry_point, config): super(BalancePlugin, self).__init__(entry_point, config) diff --git a/flaschengeist/plugins/balance/migrations/98f2733bbe45_balance_initial.py b/flaschengeist/plugins/balance/migrations/98f2733bbe45_balance_initial.py new file mode 100644 index 0000000..2a9d322 --- /dev/null +++ b/flaschengeist/plugins/balance/migrations/98f2733bbe45_balance_initial.py @@ -0,0 +1,47 @@ +"""balance: initial + +Revision ID: 98f2733bbe45 +Revises: +Create Date: 2022-02-23 14:41:03.089145 + +""" +from alembic import op +import sqlalchemy as sa +import flaschengeist + + +# revision identifiers, used by Alembic. +revision = "98f2733bbe45" +down_revision = None +branch_labels = ("balance",) +depends_on = "flaschengeist" + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "balance_transaction", + sa.Column("receiver_id", flaschengeist.models.Serial(), nullable=True), + sa.Column("sender_id", flaschengeist.models.Serial(), nullable=True), + sa.Column("author_id", flaschengeist.models.Serial(), nullable=False), + sa.Column("id", flaschengeist.models.Serial(), nullable=False), + sa.Column("time", flaschengeist.models.UtcDateTime(), nullable=False), + sa.Column("amount", sa.Numeric(precision=5, scale=2, asdecimal=False), nullable=False), + sa.Column("reversal_id", flaschengeist.models.Serial(), nullable=True), + sa.ForeignKeyConstraint(["author_id"], ["user.id"], name=op.f("fk_balance_transaction_author_id_user")), + sa.ForeignKeyConstraint(["receiver_id"], ["user.id"], name=op.f("fk_balance_transaction_receiver_id_user")), + sa.ForeignKeyConstraint( + ["reversal_id"], + ["balance_transaction.id"], + name=op.f("fk_balance_transaction_reversal_id_balance_transaction"), + ), + sa.ForeignKeyConstraint(["sender_id"], ["user.id"], name=op.f("fk_balance_transaction_sender_id_user")), + sa.PrimaryKeyConstraint("id", name=op.f("pk_balance_transaction")), + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("balance_transaction") + # ### end Alembic commands ### diff --git a/flaschengeist/plugins/message_mail.py b/flaschengeist/plugins/message_mail.py index 3a9502a..2fe6a76 100644 --- a/flaschengeist/plugins/message_mail.py +++ b/flaschengeist/plugins/message_mail.py @@ -13,7 +13,7 @@ from . import Plugin class MailMessagePlugin(Plugin): def __init__(self, entry_point, config): - super().__init__(entry_point) + super().__init__(entry_point, config) self.server = config["SERVER"] self.port = config["PORT"] self.user = config["USER"] diff --git a/flaschengeist/plugins/pricelist/__init__.py b/flaschengeist/plugins/pricelist/__init__.py index d06af7f..3f45351 100644 --- a/flaschengeist/plugins/pricelist/__init__.py +++ b/flaschengeist/plugins/pricelist/__init__.py @@ -1,5 +1,6 @@ """Pricelist plugin""" +import pathlib from flask import Blueprint, jsonify, request, current_app from werkzeug.local import LocalProxy from werkzeug.exceptions import BadRequest, Forbidden, NotFound, Unauthorized @@ -26,6 +27,7 @@ class PriceListPlugin(Plugin): def __init__(self, entry_point, config=None): super().__init__(entry_point, config) self.blueprint = blueprint + self.migrations_path = (pathlib.Path(__file__).parent / "migrations").resolve() config = {"discount": 0} config.update(config) diff --git a/flaschengeist/plugins/pricelist/migrations/58ab9b6a8839_pricelist_initial.py b/flaschengeist/plugins/pricelist/migrations/58ab9b6a8839_pricelist_initial.py new file mode 100644 index 0000000..3a6c5ad --- /dev/null +++ b/flaschengeist/plugins/pricelist/migrations/58ab9b6a8839_pricelist_initial.py @@ -0,0 +1,141 @@ +"""pricelist: initial + +Revision ID: 58ab9b6a8839 +Revises: +Create Date: 2022-02-23 14:45:30.563647 + +""" +from alembic import op +import sqlalchemy as sa +import flaschengeist + + +# revision identifiers, used by Alembic. +revision = "58ab9b6a8839" +down_revision = None +branch_labels = ("pricelist",) +depends_on = "flaschengeist" + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "drink_extra_ingredient", + sa.Column("id", flaschengeist.models.Serial(), nullable=False), + sa.Column("name", sa.String(length=30), nullable=False), + sa.Column("price", sa.Numeric(precision=5, scale=2, asdecimal=False), nullable=True), + sa.PrimaryKeyConstraint("id", name=op.f("pk_drink_extra_ingredient")), + sa.UniqueConstraint("name", name=op.f("uq_drink_extra_ingredient_name")), + ) + op.create_table( + "drink_tag", + sa.Column("id", flaschengeist.models.Serial(), nullable=False), + sa.Column("name", sa.String(length=30), nullable=False), + sa.Column("color", sa.String(length=7), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_drink_tag")), + sa.UniqueConstraint("name", name=op.f("uq_drink_tag_name")), + ) + op.create_table( + "drink_type", + sa.Column("id", flaschengeist.models.Serial(), nullable=False), + sa.Column("name", sa.String(length=30), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_drink_type")), + sa.UniqueConstraint("name", name=op.f("uq_drink_type_name")), + ) + op.create_table( + "drink", + sa.Column("id", flaschengeist.models.Serial(), nullable=False), + sa.Column("article_id", sa.String(length=64), nullable=True), + sa.Column("package_size", sa.Integer(), nullable=True), + sa.Column("name", sa.String(length=60), nullable=False), + sa.Column("volume", sa.Numeric(precision=5, scale=2, asdecimal=False), nullable=True), + sa.Column("cost_per_volume", sa.Numeric(precision=5, scale=3, asdecimal=False), nullable=True), + sa.Column("cost_per_package", sa.Numeric(precision=5, scale=3, asdecimal=False), nullable=True), + sa.Column("receipt", sa.PickleType(), nullable=True), + sa.Column("type_id", flaschengeist.models.Serial(), nullable=True), + sa.Column("image_id", flaschengeist.models.Serial(), nullable=True), + sa.ForeignKeyConstraint(["image_id"], ["image.id"], name=op.f("fk_drink_image_id_image")), + sa.ForeignKeyConstraint(["type_id"], ["drink_type.id"], name=op.f("fk_drink_type_id_drink_type")), + sa.PrimaryKeyConstraint("id", name=op.f("pk_drink")), + ) + op.create_table( + "drink_ingredient", + sa.Column("id", flaschengeist.models.Serial(), nullable=False), + sa.Column("volume", sa.Numeric(precision=5, scale=2, asdecimal=False), nullable=False), + sa.Column("ingredient_id", flaschengeist.models.Serial(), nullable=True), + sa.ForeignKeyConstraint(["ingredient_id"], ["drink.id"], name=op.f("fk_drink_ingredient_ingredient_id_drink")), + sa.PrimaryKeyConstraint("id", name=op.f("pk_drink_ingredient")), + ) + op.create_table( + "drink_price_volume", + sa.Column("id", flaschengeist.models.Serial(), nullable=False), + sa.Column("drink_id", flaschengeist.models.Serial(), nullable=True), + sa.Column("volume", sa.Numeric(precision=5, scale=2, asdecimal=False), nullable=True), + sa.ForeignKeyConstraint(["drink_id"], ["drink.id"], name=op.f("fk_drink_price_volume_drink_id_drink")), + sa.PrimaryKeyConstraint("id", name=op.f("pk_drink_price_volume")), + ) + op.create_table( + "drink_x_tag", + sa.Column("drink_id", flaschengeist.models.Serial(), nullable=True), + sa.Column("tag_id", flaschengeist.models.Serial(), nullable=True), + sa.ForeignKeyConstraint(["drink_id"], ["drink.id"], name=op.f("fk_drink_x_tag_drink_id_drink")), + sa.ForeignKeyConstraint(["tag_id"], ["drink_tag.id"], name=op.f("fk_drink_x_tag_tag_id_drink_tag")), + ) + op.create_table( + "drink_x_type", + sa.Column("drink_id", flaschengeist.models.Serial(), nullable=True), + sa.Column("type_id", flaschengeist.models.Serial(), nullable=True), + sa.ForeignKeyConstraint(["drink_id"], ["drink.id"], name=op.f("fk_drink_x_type_drink_id_drink")), + sa.ForeignKeyConstraint(["type_id"], ["drink_type.id"], name=op.f("fk_drink_x_type_type_id_drink_type")), + ) + op.create_table( + "drink_ingredient_association", + sa.Column("id", flaschengeist.models.Serial(), nullable=False), + sa.Column("volume_id", flaschengeist.models.Serial(), nullable=True), + sa.Column("_drink_ingredient_id", flaschengeist.models.Serial(), nullable=True), + sa.Column("_extra_ingredient_id", flaschengeist.models.Serial(), nullable=True), + sa.ForeignKeyConstraint( + ["_drink_ingredient_id"], + ["drink_ingredient.id"], + name=op.f("fk_drink_ingredient_association__drink_ingredient_id_drink_ingredient"), + ), + sa.ForeignKeyConstraint( + ["_extra_ingredient_id"], + ["drink_extra_ingredient.id"], + name=op.f("fk_drink_ingredient_association__extra_ingredient_id_drink_extra_ingredient"), + ), + sa.ForeignKeyConstraint( + ["volume_id"], + ["drink_price_volume.id"], + name=op.f("fk_drink_ingredient_association_volume_id_drink_price_volume"), + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_drink_ingredient_association")), + ) + op.create_table( + "drink_price", + sa.Column("id", flaschengeist.models.Serial(), nullable=False), + sa.Column("price", sa.Numeric(precision=5, scale=2, asdecimal=False), nullable=True), + sa.Column("volume_id", flaschengeist.models.Serial(), nullable=True), + sa.Column("public", sa.Boolean(), nullable=True), + sa.Column("description", sa.String(length=30), nullable=True), + sa.ForeignKeyConstraint( + ["volume_id"], ["drink_price_volume.id"], name=op.f("fk_drink_price_volume_id_drink_price_volume") + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_drink_price")), + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("drink_price") + op.drop_table("drink_ingredient_association") + op.drop_table("drink_x_type") + op.drop_table("drink_x_tag") + op.drop_table("drink_price_volume") + op.drop_table("drink_ingredient") + op.drop_table("drink") + op.drop_table("drink_type") + op.drop_table("drink_tag") + op.drop_table("drink_extra_ingredient") + # ### end Alembic commands ### diff --git a/setup.cfg b/setup.cfg index c18feda..41af0da 100644 --- a/setup.cfg +++ b/setup.cfg @@ -22,16 +22,16 @@ include_package_data = True python_requires = >=3.9 packages = find: install_requires = - Flask >= 2.0 + Flask>=2.0 Pillow>=8.4.0 flask_cors + flask_migrate>=3.1.0 flask_sqlalchemy>=2.5 # Importlib requirement can be dropped when python requirement is >= 3.10 importlib_metadata>=4.3 sqlalchemy>=1.4.26 toml - werkzeug - + werkzeug >= 2.0 [options.extras_require] argon = argon2-cffi @@ -42,7 +42,7 @@ mysql = mysqlclient;platform_system!='Windows' [options.package_data] -* = *.toml +* = *.toml, script.py.mako [options.entry_points] console_scripts = From 4fbd20f78ecbf44750bc7f8812f86e0b14a33e7d Mon Sep 17 00:00:00 2001 From: Ferdinand Thiessen Date: Mon, 20 Dec 2021 00:53:49 +0100 Subject: [PATCH 03/27] feat(docs): Add documentation on how to install tables Also various documentation fixed and improvments --- README.md | 48 ++++++++-------- docs/plugin_development.md | 44 ++++++++++++++ flaschengeist/plugins/__init__.py | 76 ++++++++++++++++--------- flaschengeist/plugins/balance/routes.py | 4 +- flaschengeist/utils/hook.py | 12 ++++ 5 files changed, 133 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index 08aa09f..1cd3901 100644 --- a/README.md +++ b/README.md @@ -31,18 +31,35 @@ or if you want to also run the tests: pip3 install --user ".[ldap,tests]" -You will also need a MySQL driver, recommended drivers are -- `mysqlclient` -- `PyMySQL` +You will also need a MySQL driver, by default one of this is installed: +- `mysqlclient` (non Windows) +- `PyMySQL` (on Windows) -`setup.py` will try to install a matching driver. +#### Hint on MySQL driver on Windows: +If you want to use `mysqlclient` instead of `PyMySQL` (performance?) you have to follow [this guide](https://www.radishlogic.com/coding/python-3/installing-mysqldb-for-python-3-in-windows/) -#### Windows -Same as above, but if you want to use `mysqlclient` instead of `PyMySQL` (performance?) you have to follow this guide: +### Install database +The user needs to have full permissions to the database. +If not you need to create user and database manually do (or similar on Windows): -https://www.radishlogic.com/coding/python-3/installing-mysqldb-for-python-3-in-windows/ + ( + echo "CREATE DATABASE flaschengeist;" + echo "CREATE USER 'flaschengeist'@'localhost' IDENTIFIED BY 'flaschengeist';" + echo "GRANT ALL PRIVILEGES ON flaschengeist.* TO 'flaschengeist'@'localhost';" + echo "FLUSH PRIVILEGES;" + ) | sudo mysql -### Configuration +Then you can install the database tables, this will update all tables from core + all enabled plugins. +*Hint:* The same command can be later used to upgrade the database after plugins or core are updated. + + $ flaschengeist db upgrade heads + +## Plugins +To only upgrade one plugin (for example the `events` plugin): + + $ flaschengeist db upgrade events@head + +## Configuration Configuration is done within the a `flaschengeist.toml`file, you can copy the one located inside the module path (where flaschegeist is installed) or create an empty one and place it inside either: 1. `~/.config/` @@ -63,21 +80,6 @@ So you have to configure one of the following options to call flaschengeists CRO - Pros: Guaranteed execution interval, no impact on user experience (at least if you do not limit wsgi worker threads) - Cons: Uses one of the webserver threads while executing -### Database installation -The user needs to have full permissions to the database. -If not you need to create user and database manually do (or similar on Windows): - - ( - echo "CREATE DATABASE flaschengeist;" - echo "CREATE USER 'flaschengeist'@'localhost' IDENTIFIED BY 'flaschengeist';" - echo "GRANT ALL PRIVILEGES ON flaschengeist.* TO 'flaschengeist'@'localhost';" - echo "FLUSH PRIVILEGES;" - ) | sudo mysql - -Then you can install the database tables and initial entries: - - $ flaschengeist install - ### Run Flaschengeist provides a CLI, based on the flask CLI, respectivly called `flaschengeist`. diff --git a/docs/plugin_development.md b/docs/plugin_development.md index 21b69c5..ee129eb 100644 --- a/docs/plugin_development.md +++ b/docs/plugin_development.md @@ -5,9 +5,53 @@ - your_plugin/ - __init__.py - ... + - migrations/ (optional) - ... - setup.cfg The basic layout of a plugin is quite simple, you will only need the `setup.cfg` or `setup.py` and the package containing your plugin code, at lease a `__init__.py` file with your `Plugin` class. +If you use custom database tables you need to provide a `migrations` directory within your package, +see next section. + +## Database Tables / Migrations +To allow upgrades of installed plugins, the database is versioned and handled +through [Alembic](https://alembic.sqlalchemy.org/en/latest/index.html) migrations. +Each plugin, which uses custom database tables, is represented as an other base. +So you could simply follow the Alembic tutorial on [how to work with multiple bases](https://alembic.sqlalchemy.org/en/latest/branches.html#creating-a-labeled-base-revision). + +A quick overview on how to work with migrations for your plugin: + + $ flaschengeist db revision -m "Create my super plugin" \ + --head=base --branch-label=myplugin_name --version-path=your/plugin/migrations + +This would add a new base named `myplugin_name`, which should be the same as the pypi name of you plugin. +If your tables depend on an other plugin or a specific base version you could of cause add + + --depends-on=VERSION + +or + + --depends-on=other_plugin + + +### Plugin Removal and Database Tables +As generic downgrades are most often hard to write, your plugin is not required to provide such functionallity. +For Flaschengeist only instable versions provide meaningful downgrade migrations down to the latest stable version. + +So this means if you do not provide downgrades you must at lease provide a series of migrations toward removal of +the database tables in case the users wants to delete the plugin. + + (base) ----> 1.0 <----> 1.1 <----> 1.2 + | + --> removal + +After the removal step the database is stamped to to "remove" your + +## Useful Hooks +There are some predefined hooks, which might get handy for you. + +For more information, please refer to +- `flaschengeist.utils.hook.HookBefore` and +- `flaschengeist.utils.hook.HookAfter` diff --git a/flaschengeist/plugins/__init__.py b/flaschengeist/plugins/__init__.py index 49e01e1..369e117 100644 --- a/flaschengeist/plugins/__init__.py +++ b/flaschengeist/plugins/__init__.py @@ -12,49 +12,73 @@ from werkzeug.datastructures import FileStorage from flaschengeist.models.user import _Avatar, User from flaschengeist.utils.hook import HookBefore, HookAfter +__all__ = [ + "plugins_installed", + "plugins_loaded", + "before_delete_user", + "before_role_updated", + "before_update_user", + "after_role_updated", + "Plugin", + "AuthPlugin", +] + +# Documentation hacks, see https://github.com/mitmproxy/pdoc/issues/320 plugins_installed = HookAfter("plugins.installed") -"""Hook decorator for when all plugins are installed - Possible use case would be to populate the database with some presets. +plugins_installed.__doc__ = """Hook decorator for when all plugins are installed - Args: - hook_result: void (kwargs) +Possible use case would be to populate the database with some presets. """ + plugins_loaded = HookAfter("plugins.loaded") -"""Hook decorator for when all plugins are loaded - Possible use case would be to check if a specific other plugin is loaded and change own behavior +plugins_loaded.__doc__ = """Hook decorator for when all plugins are loaded - Args: - app: Current flask app instance (args) - hook_result: void (kwargs) +Possible use case would be to check if a specific other plugin is loaded and change own behavior + +Passed args: + - *app:* Current flask app instance (args) """ + before_role_updated = HookBefore("update_role") -"""Hook decorator for when roles are modified -Args: - role: Role object to modify - new_name: New name if the name was changed (None if delete) +before_role_updated.__doc__ = """Hook decorator for when roles are modified + +Passed args: + - *role:* `flaschengeist.models.user.Role` to modify + - *new_name:* New name if the name was changed (*None* if delete) """ + after_role_updated = HookAfter("update_role") -"""Hook decorator for when roles are modified -Args: - role: Role object containing the modified role - new_name: New name if the name was changed (None if deleted) +after_role_updated.__doc__ = """Hook decorator for when roles are modified + +Passed args: + - *role:* modified `flaschengeist.models.user.Role` + - *new_name:* New name if the name was changed (*None* if deleted) """ + before_update_user = HookBefore("update_user") -"""Hook decorator, when ever an user update is done, this is called before. -Args: - user: User object +before_update_user.__doc__ = """Hook decorator, when ever an user update is done, this is called before. + +Passed args: + - *user:* `flaschengeist.models.user.User` object """ + before_delete_user = HookBefore("delete_user") -"""Hook decorator,this is called before an user gets deleted. -Args: - user: User object +before_delete_user.__doc__ = """Hook decorator,this is called before an user gets deleted. + +Passed args: + - *user:* `flaschengeist.models.user.User` object """ class Plugin: """Base class for all Plugins - If your class uses custom models add a static property called ``models``. + All plugins must derived from this class. + + Optional: + - *blueprint*: `flask.Blueprint` providing your routes + - *permissions*: List of your custom permissions + - *models*: Your models, used for API export """ name: str @@ -250,14 +274,14 @@ class AuthPlugin(Plugin): """ raise NotFound - def set_avatar(self, user: User, file: FileStorage): + def set_avatar(self, user, file: FileStorage): """Set the avatar for given user (if supported by auth backend) Default behavior is to use native Image objects stored on the Flaschengeist server Args: user: User to set the avatar for - file: FileStorage object uploaded by the user + file: `werkzeug.datastructures.FileStorage` uploaded by the user Raises: MethodNotAllowed: If not supported by Backend Any valid HTTP exception diff --git a/flaschengeist/plugins/balance/routes.py b/flaschengeist/plugins/balance/routes.py index f62a065..f0edc62 100644 --- a/flaschengeist/plugins/balance/routes.py +++ b/flaschengeist/plugins/balance/routes.py @@ -134,7 +134,7 @@ def get_balance(userid, current_session: Session): Route: ``/users//balance`` | Method: ``GET`` - GET-parameters: ```{from?: string, to?: string}``` + GET-parameters: ``{from?: string, to?: string}`` Args: userid: Userid of user to get balance from @@ -173,7 +173,7 @@ def get_transactions(userid, current_session: Session): Route: ``/users//balance/transactions`` | Method: ``GET`` - GET-parameters: ```{from?: string, to?: string, limit?: int, offset?: int}``` + GET-parameters: ``{from?: string, to?: string, limit?: int, offset?: int}`` Args: userid: Userid of user to get transactions from diff --git a/flaschengeist/utils/hook.py b/flaschengeist/utils/hook.py index b028f30..f7c7fb7 100644 --- a/flaschengeist/utils/hook.py +++ b/flaschengeist/utils/hook.py @@ -7,6 +7,7 @@ _hooks_after = {} def Hook(function=None, id=None): """Hook decorator + Use to decorate functions as hooks, so plugins can hook up their custom functions. """ # `id` passed as `arg` not `kwarg` @@ -38,8 +39,10 @@ def Hook(function=None, id=None): def HookBefore(id: str): """Decorator for functions to be called before a Hook-Function is called + The hooked up function must accept the same arguments as the function hooked onto, as the functions are called with the same arguments. + Hint: This enables you to modify the arguments! """ if not id or not isinstance(id, str): @@ -54,9 +57,18 @@ def HookBefore(id: str): def HookAfter(id: str): """Decorator for functions to be called after a Hook-Function is called + As with the HookBefore, the hooked up function must accept the same arguments as the function hooked onto, but also receives a `hook_result` kwarg containing the result of the function. + + Example: + ```py + @HookAfter("some.id") + def my_func(hook_result): + # This function is executed after the function registered with "some.id" + print(hook_result) # This is the result of the function + ``` """ if not id or not isinstance(id, str): From 573bea2da029203a99daac1d884e75e3b850655c Mon Sep 17 00:00:00 2001 From: Ferdinand Thiessen Date: Thu, 23 Dec 2021 02:49:19 +0100 Subject: [PATCH 04/27] feat(cli): Added CLI command for handling plugins * Install / Uninstall plugins * List plugins --- flaschengeist/cli/__init__.py | 4 +- flaschengeist/cli/plugin_cmd.py | 96 +++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+), 2 deletions(-) create mode 100644 flaschengeist/cli/plugin_cmd.py diff --git a/flaschengeist/cli/__init__.py b/flaschengeist/cli/__init__.py index e86e60c..ed93b5d 100644 --- a/flaschengeist/cli/__init__.py +++ b/flaschengeist/cli/__init__.py @@ -84,7 +84,7 @@ def cli(): def main(*args, **kwargs): - # from .plugin_cmd import plugin + from .plugin_cmd import plugin from .export_cmd import export from .docs_cmd import docs from .run_cmd import run @@ -92,8 +92,8 @@ def main(*args, **kwargs): # Override logging level environ.setdefault("FG_LOGGING", logging.getLevelName(LOGGING_MAX)) - # cli.add_command(plugin) cli.add_command(export) cli.add_command(docs) + cli.add_command(plugin) cli.add_command(run) cli(*args, **kwargs) diff --git a/flaschengeist/cli/plugin_cmd.py b/flaschengeist/cli/plugin_cmd.py new file mode 100644 index 0000000..bb0dd98 --- /dev/null +++ b/flaschengeist/cli/plugin_cmd.py @@ -0,0 +1,96 @@ +import click +from click.decorators import pass_context +from flask import current_app +from flask.cli import with_appcontext +from importlib_metadata import EntryPoint, entry_points + +from flaschengeist.config import config + + +@click.group() +def plugin(): + pass + + +@plugin.command() +@click.argument("plugin", nargs=-1, type=str) +@click.option("--all", help="Install all enabled plugins", is_flag=True) +@with_appcontext +@pass_context +def install(ctx, plugin, all): + """Install one or more plugins""" + if not all and len(plugin) == 0: + ctx.fail("At least one plugin must be specified, or use `--all` flag.") + if all: + plugins = current_app.config["FG_PLUGINS"].values() + else: + try: + plugins = [current_app.config["FG_PLUGINS"][p] for p in plugin] + except KeyError as e: + ctx.fail(f"Invalid plugin ID, could not find >{e.args[0]}<") + for p in plugins: + name = p.id.split(".")[-1] + click.echo(f"Installing {name}{'.'*(20-len(name))}", nl=False) + p.install() + click.secho(" ok", fg="green") + + +@plugin.command() +@click.argument("plugin", nargs=-1, type=str) +@with_appcontext +@pass_context +def uninstall(ctx: click.Context, plugin): + """Uninstall one or more plugins""" + + if len(plugin) == 0: + ctx.fail("At least one plugin must be specified") + try: + plugins = [current_app.config["FG_PLUGINS"][p] for p in plugin] + except KeyError as e: + ctx.fail(f"Invalid plugin ID, could not find >{e.args[0]}<") + if ( + click.prompt( + "You are going to uninstall:\n\n" + f"\t{', '.join([p.id.split('.')[-1] for p in plugins])}\n\n" + "Are you sure?", + default="n", + show_choices=True, + type=click.Choice(["y", "N"], False), + ).lower() + != "y" + ): + ctx.exit() + for p in plugins: + name = p.id.split(".")[-1] + click.echo(f"Uninstalling {name}{'.'*(20-len(name))}", nl=False) + p.uninstall() + click.secho(" ok", fg="green") + + +@plugin.command() +@click.option("--enabled", "-e", help="List only enabled plugins", is_flag=True) +@click.option("--no-header", "-n", help="Do not show header", is_flag=True) +@with_appcontext +def ls(enabled, no_header): + def plugin_version(p): + if isinstance(p, EntryPoint): + return p.dist.version + return p.version + + plugins = entry_points(group="flaschengeist.plugins") + enabled_plugins = [key for key, value in config.items() if "enabled" in value] + [config["FLASCHENGEIST"]["auth"]] + loaded_plugins = current_app.config["FG_PLUGINS"].keys() + + if not no_header: + print(f"{' '*13}{'name': <20}|{'version': >10}") + print("-" * 46) + for plugin in plugins: + if enabled and plugin.name not in enabled_plugins: + continue + print( + f"{plugin.name: <33}|{plugin_version(plugin): >12}" + f"{click.style(' (enabled)', fg='green') if plugin.name in enabled_plugins else ' (disabled)'}" + ) + + for plugin in [value for value in enabled_plugins if value not in loaded_plugins]: + print(f"{plugin: <33}|{' '*12}" f"{click.style(' (not found)', fg='red')}") From f1d6b6a2f2b6600cb2f572fc1ce170873b5c5eb9 Mon Sep 17 00:00:00 2001 From: Ferdinand Thiessen Date: Sun, 31 Jul 2022 13:22:11 +0200 Subject: [PATCH 05/27] [plugins][cli] Fix initial migration file + Make sure plugin permissions are installed Signed-off-by: Ferdinand Thiessen --- .../255b93b6beed_flaschengeist_initial.py | 17 ++++-- flaschengeist/cli/plugin_cmd.py | 59 ++++++++++++------- flaschengeist/models/image.py | 10 ++-- flaschengeist/models/setting.py | 4 +- flaschengeist/plugins/__init__.py | 3 +- flaschengeist/plugins/auth/__init__.py | 1 - flaschengeist/plugins/auth_plain/__init__.py | 4 +- 7 files changed, 61 insertions(+), 37 deletions(-) diff --git a/flaschengeist/alembic/migrations/255b93b6beed_flaschengeist_initial.py b/flaschengeist/alembic/migrations/255b93b6beed_flaschengeist_initial.py index b7deac6..b18a71e 100644 --- a/flaschengeist/alembic/migrations/255b93b6beed_flaschengeist_initial.py +++ b/flaschengeist/alembic/migrations/255b93b6beed_flaschengeist_initial.py @@ -18,14 +18,21 @@ depends_on = None def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "plugin_setting", + sa.Column("id", flaschengeist.models.Serial(), nullable=False), + sa.Column("plugin", sa.String(length=127), nullable=True), + sa.Column("name", sa.String(length=127), nullable=False), + sa.Column("value", sa.PickleType(protocol=4), nullable=True), + sa.PrimaryKeyConstraint("id", name=op.f("pk_plugin_setting")), + ) op.create_table( "image", sa.Column("id", flaschengeist.models.Serial(), nullable=False), - sa.Column("filename_", sa.String(length=127), nullable=False), - sa.Column("mimetype_", sa.String(length=30), nullable=False), - sa.Column("thumbnail_", sa.String(length=127), nullable=True), - sa.Column("path_", sa.String(length=127), nullable=True), + sa.Column("filename", sa.String(length=255), nullable=False), + sa.Column("mimetype", sa.String(length=127), nullable=False), + sa.Column("thumbnail", sa.String(length=255), nullable=True), + sa.Column("path", sa.String(length=255), nullable=True), sa.PrimaryKeyConstraint("id", name=op.f("pk_image")), ) op.create_table( diff --git a/flaschengeist/cli/plugin_cmd.py b/flaschengeist/cli/plugin_cmd.py index bb0dd98..c2b5274 100644 --- a/flaschengeist/cli/plugin_cmd.py +++ b/flaschengeist/cli/plugin_cmd.py @@ -4,7 +4,9 @@ from flask import current_app from flask.cli import with_appcontext from importlib_metadata import EntryPoint, entry_points +from flaschengeist.database import db from flaschengeist.config import config +from flaschengeist.models.user import Permission @click.group() @@ -12,6 +14,35 @@ def plugin(): pass +def install_plugin_command(ctx, plugin, all): + """Install one or more plugins""" + if not all and len(plugin) == 0: + ctx.fail("At least one plugin must be specified, or use `--all` flag.") + + if all: + plugins = current_app.config["FG_PLUGINS"] + else: + try: + plugins = {plugin_name: current_app.config["FG_PLUGINS"][plugin_name] for plugin_name in plugin} + except KeyError as e: + ctx.fail(f"Invalid plugin name, could not find >{e.args[0]}<") + + for name, plugin in plugins.items(): + click.echo(f"Installing {name}{'.'*(20-len(name))}", nl=False) + # Install permissions + if plugin.permissions: + cur_perm = set(x.name for x in Permission.query.filter(Permission.name.in_(plugin.permissions)).all()) + all_perm = set(plugin.permissions) + + add = all_perm - cur_perm + if add: + db.session.bulk_save_objects([Permission(name=x) for x in all_perm]) + db.session.commit() + # Custom installation steps + plugin.install() + click.secho(" ok", fg="green") + + @plugin.command() @click.argument("plugin", nargs=-1, type=str) @click.option("--all", help="Install all enabled plugins", is_flag=True) @@ -19,20 +50,7 @@ def plugin(): @pass_context def install(ctx, plugin, all): """Install one or more plugins""" - if not all and len(plugin) == 0: - ctx.fail("At least one plugin must be specified, or use `--all` flag.") - if all: - plugins = current_app.config["FG_PLUGINS"].values() - else: - try: - plugins = [current_app.config["FG_PLUGINS"][p] for p in plugin] - except KeyError as e: - ctx.fail(f"Invalid plugin ID, could not find >{e.args[0]}<") - for p in plugins: - name = p.id.split(".")[-1] - click.echo(f"Installing {name}{'.'*(20-len(name))}", nl=False) - p.install() - click.secho(" ok", fg="green") + return install_plugin_command(ctx, plugin, all) @plugin.command() @@ -44,14 +62,16 @@ def uninstall(ctx: click.Context, plugin): if len(plugin) == 0: ctx.fail("At least one plugin must be specified") + try: - plugins = [current_app.config["FG_PLUGINS"][p] for p in plugin] + plugins = {plugin_name: current_app.config["FG_PLUGINS"][plugin_name] for plugin_name in plugin} except KeyError as e: ctx.fail(f"Invalid plugin ID, could not find >{e.args[0]}<") + if ( click.prompt( "You are going to uninstall:\n\n" - f"\t{', '.join([p.id.split('.')[-1] for p in plugins])}\n\n" + f"\t{', '.join([plugin_name for plugin_name in plugins.keys()])}\n\n" "Are you sure?", default="n", show_choices=True, @@ -60,10 +80,9 @@ def uninstall(ctx: click.Context, plugin): != "y" ): ctx.exit() - for p in plugins: - name = p.id.split(".")[-1] + for name, plugin in plugins.items(): click.echo(f"Uninstalling {name}{'.'*(20-len(name))}", nl=False) - p.uninstall() + plugin.uninstall() click.secho(" ok", fg="green") @@ -78,7 +97,7 @@ def ls(enabled, no_header): return p.version plugins = entry_points(group="flaschengeist.plugins") - enabled_plugins = [key for key, value in config.items() if "enabled" in value] + [config["FLASCHENGEIST"]["auth"]] + enabled_plugins = [key for key, value in config.items() if "enabled" in value and value["enabled"]] + [config["FLASCHENGEIST"]["auth"]] loaded_plugins = current_app.config["FG_PLUGINS"].keys() if not no_header: diff --git a/flaschengeist/models/image.py b/flaschengeist/models/image.py index 4c963e7..9a97ea8 100644 --- a/flaschengeist/models/image.py +++ b/flaschengeist/models/image.py @@ -9,11 +9,11 @@ from ..database import db class Image(db.Model, ModelSerializeMixin): __tablename__ = "image" - id: int = db.Column("id", Serial, primary_key=True) - filename_: str = db.Column(db.String(127), nullable=False) - mimetype_: str = db.Column(db.String(30), nullable=False) - thumbnail_: str = db.Column(db.String(127)) - path_: str = db.Column(db.String(127)) + id: int = db.Column(Serial, primary_key=True) + filename_: str = db.Column("filename", db.String(255), nullable=False) + mimetype_: str = db.Column("mimetype", db.String(127), nullable=False) + thumbnail_: str = db.Column("thumbnail", db.String(255)) + path_: str = db.Column("path", db.String(255)) def open(self): return open(self.path_, "rb") diff --git a/flaschengeist/models/setting.py b/flaschengeist/models/setting.py index 277f36c..b090c3e 100644 --- a/flaschengeist/models/setting.py +++ b/flaschengeist/models/setting.py @@ -8,6 +8,6 @@ from ..database import db class _PluginSetting(db.Model): __tablename__ = "plugin_setting" id = db.Column("id", Serial, primary_key=True) - plugin: str = db.Column(db.String(30)) - name: str = db.Column(db.String(30), nullable=False) + plugin: str = db.Column(db.String(127)) + name: str = db.Column(db.String(127), nullable=False) value: Any = db.Column(db.PickleType(protocol=4)) diff --git a/flaschengeist/plugins/__init__.py b/flaschengeist/plugins/__init__.py index 369e117..13936e5 100644 --- a/flaschengeist/plugins/__init__.py +++ b/flaschengeist/plugins/__init__.py @@ -126,7 +126,8 @@ class Plugin: def install(self): """Installation routine - Is always called with Flask application context + Is always called with Flask application context, + it is called after the plugin permissions are installed. """ pass diff --git a/flaschengeist/plugins/auth/__init__.py b/flaschengeist/plugins/auth/__init__.py index 66d77be..afcc854 100644 --- a/flaschengeist/plugins/auth/__init__.py +++ b/flaschengeist/plugins/auth/__init__.py @@ -13,7 +13,6 @@ from flaschengeist.controller import sessionController, userController class AuthRoutePlugin(Plugin): - id = "dev.flaschengeist.auth" blueprint = Blueprint("auth", __name__) diff --git a/flaschengeist/plugins/auth_plain/__init__.py b/flaschengeist/plugins/auth_plain/__init__.py index e73b98c..44c27f7 100644 --- a/flaschengeist/plugins/auth_plain/__init__.py +++ b/flaschengeist/plugins/auth_plain/__init__.py @@ -14,12 +14,10 @@ from flaschengeist import logger class AuthPlain(AuthPlugin): - id = "auth_plain" - def install(self): plugins_installed(self.post_install) - def post_install(self, **kwargs): + def post_install(self, *args, **kwargs): if User.query.filter(User.deleted == False).count() == 0: logger.info("Installing admin user") role = Role.query.filter(Role.name == "Superuser").first() From fa503fe142311246baf3a4a5bfa2ddbdea7de155 Mon Sep 17 00:00:00 2001 From: Ferdinand Thiessen Date: Sun, 31 Jul 2022 15:48:47 +0200 Subject: [PATCH 06/27] [cli] Added install command to install the database and all plugins Signed-off-by: Ferdinand Thiessen --- README.md | 9 ++++++++- flaschengeist/alembic/__init__.py | 4 ++++ flaschengeist/cli/__init__.py | 2 ++ flaschengeist/cli/install_cmd.py | 20 ++++++++++++++++++++ flaschengeist/plugins/users/__init__.py | 2 +- 5 files changed, 35 insertions(+), 2 deletions(-) create mode 100644 flaschengeist/cli/install_cmd.py diff --git a/README.md b/README.md index 1cd3901..320bed5 100644 --- a/README.md +++ b/README.md @@ -50,10 +50,16 @@ If not you need to create user and database manually do (or similar on Windows): ) | sudo mysql Then you can install the database tables, this will update all tables from core + all enabled plugins. -*Hint:* The same command can be later used to upgrade the database after plugins or core are updated. +And also install all enabled plugins: + + $ flaschengeist install + +*Hint:* To only install the database tables, or upgrade the database after plugins or core are updated later +you can use this command: $ flaschengeist db upgrade heads + ## Plugins To only upgrade one plugin (for example the `events` plugin): @@ -88,6 +94,7 @@ with the difference of the main logger will be forced to output to `stderr` and of the CLI will override the logging level you have configured for the main logger. $ flaschengeist run + or with debug messages: $ flaschengeist run --debug diff --git a/flaschengeist/alembic/__init__.py b/flaschengeist/alembic/__init__.py index e69de29..cf71018 100644 --- a/flaschengeist/alembic/__init__.py +++ b/flaschengeist/alembic/__init__.py @@ -0,0 +1,4 @@ +from pathlib import Path + + +alembic_migrations = str(Path(__file__).resolve().parent / "migrations") diff --git a/flaschengeist/cli/__init__.py b/flaschengeist/cli/__init__.py index ed93b5d..49e3333 100644 --- a/flaschengeist/cli/__init__.py +++ b/flaschengeist/cli/__init__.py @@ -88,12 +88,14 @@ def main(*args, **kwargs): from .export_cmd import export from .docs_cmd import docs from .run_cmd import run + from .install_cmd import install # Override logging level environ.setdefault("FG_LOGGING", logging.getLevelName(LOGGING_MAX)) cli.add_command(export) cli.add_command(docs) + cli.add_command(install) cli.add_command(plugin) cli.add_command(run) cli(*args, **kwargs) diff --git a/flaschengeist/cli/install_cmd.py b/flaschengeist/cli/install_cmd.py new file mode 100644 index 0000000..3d1b1ff --- /dev/null +++ b/flaschengeist/cli/install_cmd.py @@ -0,0 +1,20 @@ +import click +from click.decorators import pass_context +from flask.cli import with_appcontext +from flask_migrate import upgrade + +from flaschengeist.alembic import alembic_migrations +from flaschengeist.cli.plugin_cmd import install_plugin_command +from flaschengeist.utils.hook import Hook + + +@click.command() +@with_appcontext +@pass_context +@Hook("plugins.installed") +def install(ctx): + # Install database + upgrade(alembic_migrations, revision="heads") + + # Install plugins + install_plugin_command(ctx, [], True) diff --git a/flaschengeist/plugins/users/__init__.py b/flaschengeist/plugins/users/__init__.py index 2e0802c..778c819 100644 --- a/flaschengeist/plugins/users/__init__.py +++ b/flaschengeist/plugins/users/__init__.py @@ -104,7 +104,7 @@ def frontend(userid, current_session): raise Forbidden if request.method == "POST": - if request.content_length > 1024**2: + if request.content_length > 1024 ** 2: raise BadRequest current_session.user_.set_attribute("frontend", request.get_json()) return no_content() From dc2b949225c290d6734dcc278c229661faf690d7 Mon Sep 17 00:00:00 2001 From: Ferdinand Thiessen Date: Sun, 31 Jul 2022 19:06:55 +0200 Subject: [PATCH 07/27] [cli] Fix exporting of plugin interfaces Signed-off-by: Ferdinand Thiessen --- flaschengeist/cli/export_cmd.py | 16 ++++++++++++---- flaschengeist/cli/plugin_cmd.py | 4 +++- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/flaschengeist/cli/export_cmd.py b/flaschengeist/cli/export_cmd.py index 6f93c70..2131f7b 100644 --- a/flaschengeist/cli/export_cmd.py +++ b/flaschengeist/cli/export_cmd.py @@ -1,4 +1,5 @@ import click +from importlib_metadata import entry_points @click.command() @@ -8,14 +9,21 @@ import click @click.option("--no-core", help="Skip models / types from flaschengeist core", is_flag=True) def export(namespace, output, no_core, plugin): from flaschengeist import logger, models - from flaschengeist.app import get_plugins from .InterfaceGenerator import InterfaceGenerator gen = InterfaceGenerator(namespace, output, logger) if not no_core: gen.run(models) if plugin: - for plugin_class in get_plugins(): - if (len(plugin) == 0 or plugin_class.id in plugin) and plugin_class.models is not None: - gen.run(plugin_class.models) + for entry_point in entry_points(group="flaschengeist.plugins"): + if len(plugin) == 0 or entry_point.name in plugin: + try: + plugin = entry_point.load() + gen.run(plugin.models) + except: + logger.error( + f"Plugin {entry_point.name} could not be loaded due to an error.", + exc_info=True, + ) + continue gen.write() diff --git a/flaschengeist/cli/plugin_cmd.py b/flaschengeist/cli/plugin_cmd.py index c2b5274..4a34cc6 100644 --- a/flaschengeist/cli/plugin_cmd.py +++ b/flaschengeist/cli/plugin_cmd.py @@ -97,7 +97,9 @@ def ls(enabled, no_header): return p.version plugins = entry_points(group="flaschengeist.plugins") - enabled_plugins = [key for key, value in config.items() if "enabled" in value and value["enabled"]] + [config["FLASCHENGEIST"]["auth"]] + enabled_plugins = [key for key, value in config.items() if "enabled" in value and value["enabled"]] + [ + config["FLASCHENGEIST"]["auth"] + ] loaded_plugins = current_app.config["FG_PLUGINS"].keys() if not no_header: From 7f8aa80b0e50614b9cebc68950a7f717dbc648da Mon Sep 17 00:00:00 2001 From: Ferdinand Thiessen Date: Thu, 18 Aug 2022 19:16:29 +0200 Subject: [PATCH 08/27] Update dependencies and increase python version to 3.10 Drop future imports, not needed with python 3.10 Signed-off-by: Ferdinand Thiessen --- flaschengeist/app.py | 2 +- flaschengeist/cli/export_cmd.py | 2 +- flaschengeist/cli/plugin_cmd.py | 2 +- flaschengeist/database.py | 4 +--- flaschengeist/models/image.py | 2 -- flaschengeist/models/notification.py | 1 - flaschengeist/models/session.py | 6 ------ flaschengeist/models/user.py | 3 --- flaschengeist/plugins/__init__.py | 2 +- flaschengeist/plugins/balance/models.py | 2 -- flaschengeist/plugins/pricelist/models.py | 2 -- setup.cfg | 14 ++++++-------- 12 files changed, 11 insertions(+), 31 deletions(-) diff --git a/flaschengeist/app.py b/flaschengeist/app.py index b6f87e1..4aa8b88 100644 --- a/flaschengeist/app.py +++ b/flaschengeist/app.py @@ -4,7 +4,7 @@ from flask import Flask from flask_cors import CORS from datetime import datetime, date from flask.json import JSONEncoder, jsonify -from importlib_metadata import entry_points +from importlib.metadata import entry_points from sqlalchemy.exc import OperationalError from werkzeug.exceptions import HTTPException diff --git a/flaschengeist/cli/export_cmd.py b/flaschengeist/cli/export_cmd.py index 2131f7b..4e0fa03 100644 --- a/flaschengeist/cli/export_cmd.py +++ b/flaschengeist/cli/export_cmd.py @@ -1,5 +1,5 @@ import click -from importlib_metadata import entry_points +from importlib.metadata import entry_points @click.command() diff --git a/flaschengeist/cli/plugin_cmd.py b/flaschengeist/cli/plugin_cmd.py index 4a34cc6..97bd1bc 100644 --- a/flaschengeist/cli/plugin_cmd.py +++ b/flaschengeist/cli/plugin_cmd.py @@ -2,7 +2,7 @@ import click from click.decorators import pass_context from flask import current_app from flask.cli import with_appcontext -from importlib_metadata import EntryPoint, entry_points +from importlib.metadata import EntryPoint, entry_points from flaschengeist.database import db from flaschengeist.config import config diff --git a/flaschengeist/database.py b/flaschengeist/database.py index a2c4672..5bb30ef 100644 --- a/flaschengeist/database.py +++ b/flaschengeist/database.py @@ -1,7 +1,7 @@ import os from flask_migrate import Migrate, Config from flask_sqlalchemy import SQLAlchemy -from importlib_metadata import EntryPoint +from importlib.metadata import EntryPoint, entry_points, distribution from sqlalchemy import MetaData from flaschengeist import logger @@ -30,8 +30,6 @@ def configure_alembic(config: Config): This includes even disabled plugins, as simply disabling a plugin without uninstall can break the alembic version management. """ - from importlib_metadata import entry_points, distribution - # Set main script location config.set_main_option( "script_location", str(distribution("flaschengeist").locate_file("") / "flaschengeist" / "alembic") diff --git a/flaschengeist/models/image.py b/flaschengeist/models/image.py index 9a97ea8..d87af8a 100644 --- a/flaschengeist/models/image.py +++ b/flaschengeist/models/image.py @@ -1,5 +1,3 @@ -from __future__ import annotations # TODO: Remove if python requirement is >= 3.10 - from sqlalchemy import event from pathlib import Path diff --git a/flaschengeist/models/notification.py b/flaschengeist/models/notification.py index 9431c17..07320c7 100644 --- a/flaschengeist/models/notification.py +++ b/flaschengeist/models/notification.py @@ -1,4 +1,3 @@ -from __future__ import annotations # TODO: Remove if python requirement is >= 3.10 from datetime import datetime from typing import Any diff --git a/flaschengeist/models/session.py b/flaschengeist/models/session.py index 9acf27c..7dc6df8 100644 --- a/flaschengeist/models/session.py +++ b/flaschengeist/models/session.py @@ -1,10 +1,4 @@ -from __future__ import annotations # TODO: Remove if python requirement is >= 3.10 - from datetime import datetime, timedelta, timezone - -from . import ModelSerializeMixin, UtcDateTime, Serial -from .user import User -from flaschengeist.database import db from secrets import compare_digest from flaschengeist import logger diff --git a/flaschengeist/models/user.py b/flaschengeist/models/user.py index 2ce1716..2889eeb 100644 --- a/flaschengeist/models/user.py +++ b/flaschengeist/models/user.py @@ -1,6 +1,3 @@ -from __future__ import annotations # TODO: Remove if python requirement is >= 3.10 - -from flask import url_for from typing import Optional from datetime import date, datetime from sqlalchemy.orm.collections import attribute_mapped_collection diff --git a/flaschengeist/plugins/__init__.py b/flaschengeist/plugins/__init__.py index 13936e5..fadecff 100644 --- a/flaschengeist/plugins/__init__.py +++ b/flaschengeist/plugins/__init__.py @@ -5,7 +5,7 @@ """ from typing import Optional -from importlib_metadata import Distribution, EntryPoint +from importlib.metadata import Distribution, EntryPoint from werkzeug.exceptions import MethodNotAllowed, NotFound from werkzeug.datastructures import FileStorage diff --git a/flaschengeist/plugins/balance/models.py b/flaschengeist/plugins/balance/models.py index 1ba206b..d5d0061 100644 --- a/flaschengeist/plugins/balance/models.py +++ b/flaschengeist/plugins/balance/models.py @@ -1,5 +1,3 @@ -from __future__ import annotations # TODO: Remove if python requirement is >= 3.10 - from datetime import datetime from typing import Optional from sqlalchemy.ext.hybrid import hybrid_property diff --git a/flaschengeist/plugins/pricelist/models.py b/flaschengeist/plugins/pricelist/models.py index 630766d..543fee0 100644 --- a/flaschengeist/plugins/pricelist/models.py +++ b/flaschengeist/plugins/pricelist/models.py @@ -1,5 +1,3 @@ -from __future__ import annotations # TODO: Remove if python requirement is >= 3.10 - from flaschengeist.database import db from flaschengeist.models import ModelSerializeMixin, Serial from flaschengeist.models.image import Image diff --git a/setup.cfg b/setup.cfg index 41af0da..46433c8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -19,19 +19,17 @@ classifiers = [options] include_package_data = True -python_requires = >=3.9 +python_requires = >=3.10 packages = find: install_requires = - Flask>=2.0 - Pillow>=8.4.0 + Flask==2.0.3 + Pillow>=9.0 flask_cors flask_migrate>=3.1.0 - flask_sqlalchemy>=2.5 - # Importlib requirement can be dropped when python requirement is >= 3.10 - importlib_metadata>=4.3 - sqlalchemy>=1.4.26 + flask_sqlalchemy>=2.5.1 + sqlalchemy>=1.4.39 toml - werkzeug >= 2.0 + werkzeug==2.0.3 [options.extras_require] argon = argon2-cffi From e41be21c47dc4bbe3bed96e1afb2ecf1c00954e4 Mon Sep 17 00:00:00 2001 From: Ferdinand Thiessen Date: Thu, 18 Aug 2022 19:53:58 +0200 Subject: [PATCH 09/27] Restructure models and database import paths Signed-off-by: Ferdinand Thiessen --- flaschengeist/app.py | 1 + flaschengeist/cli/plugin_cmd.py | 2 +- flaschengeist/controller/imageController.py | 13 ++- flaschengeist/controller/messageController.py | 2 +- flaschengeist/controller/pluginController.py | 19 ++-- flaschengeist/controller/roleController.py | 8 +- flaschengeist/controller/sessionController.py | 11 +- flaschengeist/controller/userController.py | 21 ++-- .../{database.py => database/__init__.py} | 0 flaschengeist/database/types.py | 95 +++++++++++++++++ flaschengeist/models/__init__.py | 100 +----------------- flaschengeist/models/image.py | 2 +- flaschengeist/models/notification.py | 3 +- flaschengeist/models/plugin.py | 24 +++++ flaschengeist/models/session.py | 6 +- flaschengeist/models/setting.py | 13 --- flaschengeist/models/user.py | 7 +- flaschengeist/plugins/__init__.py | 11 +- flaschengeist/plugins/auth/__init__.py | 4 +- flaschengeist/plugins/auth_ldap/__init__.py | 3 +- flaschengeist/plugins/auth_plain/__init__.py | 2 +- flaschengeist/plugins/message_mail.py | 7 +- flaschengeist/plugins/pricelist/models.py | 4 +- flaschengeist/plugins/roles/__init__.py | 7 +- flaschengeist/plugins/scheduler.py | 5 +- flaschengeist/plugins/users/__init__.py | 8 +- 26 files changed, 202 insertions(+), 176 deletions(-) rename flaschengeist/{database.py => database/__init__.py} (100%) create mode 100644 flaschengeist/database/types.py create mode 100644 flaschengeist/models/plugin.py delete mode 100644 flaschengeist/models/setting.py diff --git a/flaschengeist/app.py b/flaschengeist/app.py index 4aa8b88..f992005 100644 --- a/flaschengeist/app.py +++ b/flaschengeist/app.py @@ -9,6 +9,7 @@ from sqlalchemy.exc import OperationalError from werkzeug.exceptions import HTTPException from flaschengeist import logger +from flaschengeist.models import Plugin from flaschengeist.utils.hook import Hook from flaschengeist.plugins import AuthPlugin from flaschengeist.config import config, configure_app diff --git a/flaschengeist/cli/plugin_cmd.py b/flaschengeist/cli/plugin_cmd.py index 97bd1bc..d8b90c4 100644 --- a/flaschengeist/cli/plugin_cmd.py +++ b/flaschengeist/cli/plugin_cmd.py @@ -6,7 +6,7 @@ from importlib.metadata import EntryPoint, entry_points from flaschengeist.database import db from flaschengeist.config import config -from flaschengeist.models.user import Permission +from flaschengeist.models import Permission @click.group() diff --git a/flaschengeist/controller/imageController.py b/flaschengeist/controller/imageController.py index 26bfd2d..3915bce 100644 --- a/flaschengeist/controller/imageController.py +++ b/flaschengeist/controller/imageController.py @@ -1,15 +1,14 @@ from datetime import date -from flask import send_file from pathlib import Path +from flask import send_file from PIL import Image as PImage - -from werkzeug.exceptions import NotFound, UnprocessableEntity -from werkzeug.datastructures import FileStorage from werkzeug.utils import secure_filename +from werkzeug.datastructures import FileStorage +from werkzeug.exceptions import NotFound, UnprocessableEntity -from flaschengeist.models.image import Image -from flaschengeist.database import db -from flaschengeist.config import config +from ..models import Image +from ..database import db +from ..config import config def check_mimetype(mime: str): diff --git a/flaschengeist/controller/messageController.py b/flaschengeist/controller/messageController.py index f43afc8..573a149 100644 --- a/flaschengeist/controller/messageController.py +++ b/flaschengeist/controller/messageController.py @@ -1,5 +1,5 @@ from flaschengeist.utils.hook import Hook -from flaschengeist.models.user import User, Role +from flaschengeist.models import User, Role class Message: diff --git a/flaschengeist/controller/pluginController.py b/flaschengeist/controller/pluginController.py index 7dbe678..8313935 100644 --- a/flaschengeist/controller/pluginController.py +++ b/flaschengeist/controller/pluginController.py @@ -4,9 +4,16 @@ Used by plugins for setting and notification functionality. """ import sqlalchemy + +from typing import Union +from flask import current_app +from werkzeug.exceptions import NotFound +from importlib.metadata import entry_points + +from .. import logger from ..database import db -from ..models.setting import _PluginSetting -from ..models.notification import Notification +from ..utils import Hook +from ..models import Plugin, PluginSetting, Notification def get_setting(plugin_id: str, name: str, **kwargs): @@ -23,7 +30,7 @@ def get_setting(plugin_id: str, name: str, **kwargs): """ try: setting = ( - _PluginSetting.query.filter(_PluginSetting.plugin == plugin_id).filter(_PluginSetting.name == name).one() + PluginSetting.query.filter(PluginSetting.plugin == plugin_id).filter(PluginSetting.name == name).one() ) return setting.value except sqlalchemy.orm.exc.NoResultFound: @@ -42,8 +49,8 @@ def set_setting(plugin_id: str, name: str, value): value: Value to be stored """ setting = ( - _PluginSetting.query.filter(_PluginSetting.plugin == plugin_id) - .filter(_PluginSetting.name == name) + PluginSetting.query.filter(PluginSetting.plugin == plugin_id) + .filter(PluginSetting.name == name) .one_or_none() ) if setting is not None: @@ -52,7 +59,7 @@ def set_setting(plugin_id: str, name: str, value): else: setting.value = value else: - db.session.add(_PluginSetting(plugin=plugin_id, name=name, value=value)) + db.session.add(PluginSetting(plugin=plugin_id, name=name, value=value)) db.session.commit() diff --git a/flaschengeist/controller/roleController.py b/flaschengeist/controller/roleController.py index 23528ad..eedb7c7 100644 --- a/flaschengeist/controller/roleController.py +++ b/flaschengeist/controller/roleController.py @@ -2,10 +2,10 @@ from typing import Union from sqlalchemy.exc import IntegrityError from werkzeug.exceptions import BadRequest, Conflict, NotFound -from flaschengeist import logger -from flaschengeist.models.user import Role, Permission -from flaschengeist.database import db, case_sensitive -from flaschengeist.utils.hook import Hook +from .. import logger +from ..models import Role, Permission +from ..database import db, case_sensitive +from ..utils.hook import Hook def get_all(): diff --git a/flaschengeist/controller/sessionController.py b/flaschengeist/controller/sessionController.py index 4cae005..5d5ceae 100644 --- a/flaschengeist/controller/sessionController.py +++ b/flaschengeist/controller/sessionController.py @@ -1,9 +1,12 @@ import secrets -from flaschengeist.models.session import Session -from flaschengeist.database import db -from flaschengeist import logger -from werkzeug.exceptions import Forbidden, Unauthorized + from datetime import datetime, timezone +from werkzeug.exceptions import Forbidden, Unauthorized + +from .. import logger +from ..models import Session +from ..database import db + lifetime = 1800 diff --git a/flaschengeist/controller/userController.py b/flaschengeist/controller/userController.py index bd6e4b8..610db39 100644 --- a/flaschengeist/controller/userController.py +++ b/flaschengeist/controller/userController.py @@ -1,5 +1,6 @@ -import secrets import re +import secrets + from io import BytesIO from sqlalchemy import exc from flask import current_app @@ -7,15 +8,15 @@ from datetime import datetime, timedelta, timezone from flask.helpers import send_file from werkzeug.exceptions import NotFound, BadRequest, Forbidden -from flaschengeist import logger -from flaschengeist.config import config -from flaschengeist.database import db -from flaschengeist.models.notification import Notification -from flaschengeist.utils.hook import Hook -from flaschengeist.utils.datetime import from_iso_format -from flaschengeist.utils.foreign_keys import merge_references -from flaschengeist.models.user import User, Role, _PasswordReset -from flaschengeist.controller import imageController, messageController, sessionController +from .. import logger +from ..config import config +from ..database import db +from ..models import Notification, User, Role +from ..models.user import _PasswordReset +from ..utils.hook import Hook +from ..utils.datetime import from_iso_format +from ..utils.foreign_keys import merge_references +from ..controller import imageController, messageController, sessionController def __active_users(): diff --git a/flaschengeist/database.py b/flaschengeist/database/__init__.py similarity index 100% rename from flaschengeist/database.py rename to flaschengeist/database/__init__.py diff --git a/flaschengeist/database/types.py b/flaschengeist/database/types.py new file mode 100644 index 0000000..369bb30 --- /dev/null +++ b/flaschengeist/database/types.py @@ -0,0 +1,95 @@ +import sys +import datetime + +from sqlalchemy import BigInteger, util +from sqlalchemy.dialects import mysql, sqlite +from sqlalchemy.types import DateTime, TypeDecorator + + +class ModelSerializeMixin: + """Mixin class used for models to serialize them automatically + Ignores private and protected members as well as members marked as not to publish (name ends with _) + """ + + def __is_optional(self, param): + if sys.version_info < (3, 8): + return False + + import typing + + hint = typing.get_type_hints(self.__class__)[param] + if ( + typing.get_origin(hint) is typing.Union + and len(typing.get_args(hint)) == 2 + and typing.get_args(hint)[1] is type(None) + ): + return getattr(self, param) is None + + def serialize(self): + """Serialize class to dict + Returns: + Dict of all not private or protected annotated member variables. + """ + d = { + param: getattr(self, param) + for param in self.__class__.__annotations__ + if not param.startswith("_") and not param.endswith("_") and not self.__is_optional(param) + } + if len(d) == 1: + key, value = d.popitem() + return value + return d + + def __str__(self) -> str: + return self.serialize().__str__() + + +class Serial(TypeDecorator): + """Same as MariaDB Serial used for IDs""" + + cache_ok = True + impl = BigInteger().with_variant(mysql.BIGINT(unsigned=True), "mysql").with_variant(sqlite.INTEGER(), "sqlite") + + # https://alembic.sqlalchemy.org/en/latest/autogenerate.html?highlight=custom%20column#affecting-the-rendering-of-types-themselves + def __repr__(self) -> str: + return util.generic_repr(self) + + +class UtcDateTime(TypeDecorator): + """Almost equivalent to `sqlalchemy.types.DateTime` with + ``timezone=True`` option, but it differs from that by: + + - Never silently take naive :class:`datetime.datetime`, instead it + always raise :exc:`ValueError` unless time zone aware value. + - :class:`datetime.datetime` value's :attr:`datetime.datetime.tzinfo` + is always converted to UTC. + - Unlike SQLAlchemy's built-in :class:`sqlalchemy.types.DateTime`, + it never return naive :class:`datetime.datetime`, but time zone + aware value, even with SQLite or MySQL. + """ + + cache_ok = True + impl = DateTime(timezone=True) + + @staticmethod + def current_utc(): + return datetime.datetime.now(tz=datetime.timezone.utc) + + def process_bind_param(self, value, dialect): + if value is not None: + if not isinstance(value, datetime.datetime): + raise TypeError("expected datetime.datetime, not " + repr(value)) + elif value.tzinfo is None: + raise ValueError("naive datetime is disallowed") + return value.astimezone(datetime.timezone.utc) + + def process_result_value(self, value, dialect): + if value is not None: + if value.tzinfo is not None: + value = value.astimezone(datetime.timezone.utc) + value = value.replace(tzinfo=datetime.timezone.utc) + return value + + # https://alembic.sqlalchemy.org/en/latest/autogenerate.html?highlight=custom%20column#affecting-the-rendering-of-types-themselves + def __repr__(self) -> str: + return util.generic_repr(self) diff --git a/flaschengeist/models/__init__.py b/flaschengeist/models/__init__.py index 369bb30..99a5e8d 100644 --- a/flaschengeist/models/__init__.py +++ b/flaschengeist/models/__init__.py @@ -1,95 +1,5 @@ -import sys -import datetime - -from sqlalchemy import BigInteger, util -from sqlalchemy.dialects import mysql, sqlite -from sqlalchemy.types import DateTime, TypeDecorator - - -class ModelSerializeMixin: - """Mixin class used for models to serialize them automatically - Ignores private and protected members as well as members marked as not to publish (name ends with _) - """ - - def __is_optional(self, param): - if sys.version_info < (3, 8): - return False - - import typing - - hint = typing.get_type_hints(self.__class__)[param] - if ( - typing.get_origin(hint) is typing.Union - and len(typing.get_args(hint)) == 2 - and typing.get_args(hint)[1] is type(None) - ): - return getattr(self, param) is None - - def serialize(self): - """Serialize class to dict - Returns: - Dict of all not private or protected annotated member variables. - """ - d = { - param: getattr(self, param) - for param in self.__class__.__annotations__ - if not param.startswith("_") and not param.endswith("_") and not self.__is_optional(param) - } - if len(d) == 1: - key, value = d.popitem() - return value - return d - - def __str__(self) -> str: - return self.serialize().__str__() - - -class Serial(TypeDecorator): - """Same as MariaDB Serial used for IDs""" - - cache_ok = True - impl = BigInteger().with_variant(mysql.BIGINT(unsigned=True), "mysql").with_variant(sqlite.INTEGER(), "sqlite") - - # https://alembic.sqlalchemy.org/en/latest/autogenerate.html?highlight=custom%20column#affecting-the-rendering-of-types-themselves - def __repr__(self) -> str: - return util.generic_repr(self) - - -class UtcDateTime(TypeDecorator): - """Almost equivalent to `sqlalchemy.types.DateTime` with - ``timezone=True`` option, but it differs from that by: - - - Never silently take naive :class:`datetime.datetime`, instead it - always raise :exc:`ValueError` unless time zone aware value. - - :class:`datetime.datetime` value's :attr:`datetime.datetime.tzinfo` - is always converted to UTC. - - Unlike SQLAlchemy's built-in :class:`sqlalchemy.types.DateTime`, - it never return naive :class:`datetime.datetime`, but time zone - aware value, even with SQLite or MySQL. - """ - - cache_ok = True - impl = DateTime(timezone=True) - - @staticmethod - def current_utc(): - return datetime.datetime.now(tz=datetime.timezone.utc) - - def process_bind_param(self, value, dialect): - if value is not None: - if not isinstance(value, datetime.datetime): - raise TypeError("expected datetime.datetime, not " + repr(value)) - elif value.tzinfo is None: - raise ValueError("naive datetime is disallowed") - return value.astimezone(datetime.timezone.utc) - - def process_result_value(self, value, dialect): - if value is not None: - if value.tzinfo is not None: - value = value.astimezone(datetime.timezone.utc) - value = value.replace(tzinfo=datetime.timezone.utc) - return value - - # https://alembic.sqlalchemy.org/en/latest/autogenerate.html?highlight=custom%20column#affecting-the-rendering-of-types-themselves - def __repr__(self) -> str: - return util.generic_repr(self) +from .session import * +from .user import * +from .plugin import * +from .notification import * +from .image import * \ No newline at end of file diff --git a/flaschengeist/models/image.py b/flaschengeist/models/image.py index d87af8a..406fefe 100644 --- a/flaschengeist/models/image.py +++ b/flaschengeist/models/image.py @@ -1,8 +1,8 @@ from sqlalchemy import event from pathlib import Path -from . import ModelSerializeMixin, Serial from ..database import db +from ..database.types import ModelSerializeMixin, Serial class Image(db.Model, ModelSerializeMixin): diff --git a/flaschengeist/models/notification.py b/flaschengeist/models/notification.py index 07320c7..a25efd4 100644 --- a/flaschengeist/models/notification.py +++ b/flaschengeist/models/notification.py @@ -1,9 +1,8 @@ from datetime import datetime from typing import Any -from . import Serial, UtcDateTime, ModelSerializeMixin from ..database import db -from .user import User +from ..database.types import Serial, UtcDateTime, ModelSerializeMixin class Notification(db.Model, ModelSerializeMixin): diff --git a/flaschengeist/models/plugin.py b/flaschengeist/models/plugin.py new file mode 100644 index 0000000..d2af46c --- /dev/null +++ b/flaschengeist/models/plugin.py @@ -0,0 +1,24 @@ +from typing import Any + +from ..database import db +from ..database.types import Serial + + +class Plugin(db.Model): + __tablename__ = "plugin" + id: int = db.Column("id", Serial, primary_key=True) + name: str = db.Column(db.String(127), nullable=False) + version: str = db.Column(db.String(30), nullable=False) + """The latest installed version""" + enabled: bool = db.Column(db.Boolean, default=False) + + settings_ = db.relationship("PluginSetting", cascade="all, delete") + permissions_ = db.relationship("Permission", cascade="all, delete") + + +class PluginSetting(db.Model): + __tablename__ = "plugin_setting" + id = db.Column("id", Serial, primary_key=True) + plugin_id: int = db.Column("plugin", Serial, db.ForeignKey("plugin.id")) + name: str = db.Column(db.String(127), nullable=False) + value: Any = db.Column(db.PickleType(protocol=4)) diff --git a/flaschengeist/models/session.py b/flaschengeist/models/session.py index 7dc6df8..dea7c62 100644 --- a/flaschengeist/models/session.py +++ b/flaschengeist/models/session.py @@ -1,5 +1,9 @@ from datetime import datetime, timedelta, timezone from secrets import compare_digest + +from ..database import db +from ..database.types import ModelSerializeMixin, UtcDateTime, Serial + from flaschengeist import logger @@ -22,7 +26,7 @@ class Session(db.Model, ModelSerializeMixin): _id = db.Column("id", Serial, primary_key=True) _user_id = db.Column("user_id", Serial, db.ForeignKey("user.id")) - user_: User = db.relationship("User", back_populates="sessions_") + user_: "User" = db.relationship("User", back_populates="sessions_") @property def userid(self): diff --git a/flaschengeist/models/setting.py b/flaschengeist/models/setting.py deleted file mode 100644 index b090c3e..0000000 --- a/flaschengeist/models/setting.py +++ /dev/null @@ -1,13 +0,0 @@ -from __future__ import annotations # TODO: Remove if python requirement is >= 3.10 -from typing import Any - -from . import Serial -from ..database import db - - -class _PluginSetting(db.Model): - __tablename__ = "plugin_setting" - id = db.Column("id", Serial, primary_key=True) - plugin: str = db.Column(db.String(127)) - name: str = db.Column(db.String(127), nullable=False) - value: Any = db.Column(db.PickleType(protocol=4)) diff --git a/flaschengeist/models/user.py b/flaschengeist/models/user.py index 2889eeb..c468ebf 100644 --- a/flaschengeist/models/user.py +++ b/flaschengeist/models/user.py @@ -3,9 +3,7 @@ from datetime import date, datetime from sqlalchemy.orm.collections import attribute_mapped_collection from ..database import db -from . import ModelSerializeMixin, UtcDateTime, Serial -from .image import Image - +from ..database.types import ModelSerializeMixin, UtcDateTime, Serial association_table = db.Table( "user_x_role", @@ -19,7 +17,6 @@ role_permission_association_table = db.Table( db.Column("permission_id", Serial, db.ForeignKey("permission.id")), ) - class Permission(db.Model, ModelSerializeMixin): __tablename__ = "permission" name: str = db.Column(db.String(30), unique=True) @@ -66,7 +63,7 @@ class User(db.Model, ModelSerializeMixin): sessions_: list["Session"] = db.relationship( "Session", back_populates="user_", cascade="all, delete, delete-orphan" ) - avatar_: Optional[Image] = db.relationship("Image", cascade="all, delete, delete-orphan", single_parent=True) + avatar_: Optional["Image"] = db.relationship("Image", cascade="all, delete, delete-orphan", single_parent=True) reset_requests_: list["_PasswordReset"] = db.relationship("_PasswordReset", cascade="all, delete, delete-orphan") # Private stuff for internal use diff --git a/flaschengeist/plugins/__init__.py b/flaschengeist/plugins/__init__.py index fadecff..76a35e3 100644 --- a/flaschengeist/plugins/__init__.py +++ b/flaschengeist/plugins/__init__.py @@ -9,7 +9,8 @@ from importlib.metadata import Distribution, EntryPoint from werkzeug.exceptions import MethodNotAllowed, NotFound from werkzeug.datastructures import FileStorage -from flaschengeist.models.user import _Avatar, User +from flaschengeist.models import User +from flaschengeist.models.user import _Avatar from flaschengeist.utils.hook import HookBefore, HookAfter __all__ = [ @@ -19,7 +20,7 @@ __all__ = [ "before_role_updated", "before_update_user", "after_role_updated", - "Plugin", + "BasePlugin", "AuthPlugin", ] @@ -70,7 +71,7 @@ Passed args: """ -class Plugin: +class BasePlugin: """Base class for all Plugins All plugins must derived from this class. @@ -193,10 +194,10 @@ class Plugin: return {"version": self.version, "permissions": self.permissions} -class AuthPlugin(Plugin): +class AuthPlugin(BasePlugin): """Base class for all authentification plugins - See also `Plugin` + See also `BasePlugin` """ def login(self, user, pw): diff --git a/flaschengeist/plugins/auth/__init__.py b/flaschengeist/plugins/auth/__init__.py index afcc854..439b2a6 100644 --- a/flaschengeist/plugins/auth/__init__.py +++ b/flaschengeist/plugins/auth/__init__.py @@ -6,13 +6,13 @@ from flask import Blueprint, request, jsonify from werkzeug.exceptions import Forbidden, BadRequest, Unauthorized from flaschengeist import logger -from flaschengeist.plugins import Plugin +from flaschengeist.plugins import BasePlugin from flaschengeist.utils.HTTP import no_content, created from flaschengeist.utils.decorators import login_required from flaschengeist.controller import sessionController, userController -class AuthRoutePlugin(Plugin): +class AuthRoutePlugin(BasePlugin): blueprint = Blueprint("auth", __name__) diff --git a/flaschengeist/plugins/auth_ldap/__init__.py b/flaschengeist/plugins/auth_ldap/__init__.py index ef8ecb1..bf2fb52 100644 --- a/flaschengeist/plugins/auth_ldap/__init__.py +++ b/flaschengeist/plugins/auth_ldap/__init__.py @@ -12,7 +12,8 @@ from werkzeug.datastructures import FileStorage from flaschengeist import logger from flaschengeist.controller import userController -from flaschengeist.models.user import User, Role, _Avatar +from flaschengeist.models import User, Role +from flaschengeist.models.user import _Avatar from flaschengeist.plugins import AuthPlugin, before_role_updated diff --git a/flaschengeist/plugins/auth_plain/__init__.py b/flaschengeist/plugins/auth_plain/__init__.py index 44c27f7..50ab4af 100644 --- a/flaschengeist/plugins/auth_plain/__init__.py +++ b/flaschengeist/plugins/auth_plain/__init__.py @@ -8,7 +8,7 @@ import hashlib import binascii from werkzeug.exceptions import BadRequest from flaschengeist.plugins import AuthPlugin, plugins_installed -from flaschengeist.models.user import User, Role, Permission +from flaschengeist.models import User, Role, Permission from flaschengeist.database import db from flaschengeist import logger diff --git a/flaschengeist/plugins/message_mail.py b/flaschengeist/plugins/message_mail.py index 2fe6a76..59d82d1 100644 --- a/flaschengeist/plugins/message_mail.py +++ b/flaschengeist/plugins/message_mail.py @@ -3,15 +3,14 @@ from email.mime.text import MIMEText from email.mime.multipart import MIMEMultipart from flaschengeist import logger -from flaschengeist.models.user import User +from flaschengeist.models import User +from flaschengeist.plugins import BasePlugin from flaschengeist.utils.hook import HookAfter from flaschengeist.controller import userController from flaschengeist.controller.messageController import Message -from . import Plugin - -class MailMessagePlugin(Plugin): +class MailMessagePlugin(BasePlugin): def __init__(self, entry_point, config): super().__init__(entry_point, config) self.server = config["SERVER"] diff --git a/flaschengeist/plugins/pricelist/models.py b/flaschengeist/plugins/pricelist/models.py index 543fee0..1d8dc23 100644 --- a/flaschengeist/plugins/pricelist/models.py +++ b/flaschengeist/plugins/pricelist/models.py @@ -1,6 +1,6 @@ from flaschengeist.database import db -from flaschengeist.models import ModelSerializeMixin, Serial -from flaschengeist.models.image import Image +from flaschengeist.database.types import ModelSerializeMixin, Serial +from flaschengeist.models import Image from typing import Optional diff --git a/flaschengeist/plugins/roles/__init__.py b/flaschengeist/plugins/roles/__init__.py index 4e3c92b..cd2fae4 100644 --- a/flaschengeist/plugins/roles/__init__.py +++ b/flaschengeist/plugins/roles/__init__.py @@ -5,17 +5,16 @@ Provides routes used to configure roles and permissions of users / roles. from werkzeug.exceptions import BadRequest from flask import Blueprint, request, jsonify -from http.client import NO_CONTENT -from flaschengeist.plugins import Plugin -from flaschengeist.utils.decorators import login_required +from flaschengeist.plugins import BasePlugin from flaschengeist.controller import roleController from flaschengeist.utils.HTTP import created, no_content +from flaschengeist.utils.decorators import login_required from . import permissions -class RolesPlugin(Plugin): +class RolesPlugin(BasePlugin): blueprint = Blueprint("roles", __name__) permissions = permissions.permissions diff --git a/flaschengeist/plugins/scheduler.py b/flaschengeist/plugins/scheduler.py index 7d15b69..a5e6eef 100644 --- a/flaschengeist/plugins/scheduler.py +++ b/flaschengeist/plugins/scheduler.py @@ -2,10 +2,9 @@ from flask import Blueprint from datetime import datetime, timedelta from flaschengeist import logger +from flaschengeist.plugins import BasePlugin from flaschengeist.utils.HTTP import no_content -from . import Plugin - class __Task: def __init__(self, function, **kwags): @@ -38,7 +37,7 @@ def scheduled(id: str, replace=False, **kwargs): return real_decorator -class SchedulerPlugin(Plugin): +class SchedulerPlugin(BasePlugin): def __init__(self, entry_point, config=None): super().__init__(entry_point, config) self.blueprint = Blueprint(self.name, __name__) diff --git a/flaschengeist/plugins/users/__init__.py b/flaschengeist/plugins/users/__init__.py index 778c819..3cd44df 100644 --- a/flaschengeist/plugins/users/__init__.py +++ b/flaschengeist/plugins/users/__init__.py @@ -2,14 +2,14 @@ Provides routes used to manage users """ -from http.client import NO_CONTENT, CREATED +from http.client import CREATED from flask import Blueprint, request, jsonify, make_response -from werkzeug.exceptions import BadRequest, Forbidden, MethodNotAllowed, NotFound +from werkzeug.exceptions import BadRequest, Forbidden, MethodNotAllowed from . import permissions from flaschengeist import logger from flaschengeist.config import config -from flaschengeist.plugins import Plugin +from flaschengeist.plugins import BasePlugin from flaschengeist.models.user import User from flaschengeist.utils.decorators import login_required, extract_session, headers from flaschengeist.controller import userController @@ -17,7 +17,7 @@ from flaschengeist.utils.HTTP import created, no_content from flaschengeist.utils.datetime import from_iso_format -class UsersPlugin(Plugin): +class UsersPlugin(BasePlugin): blueprint = Blueprint("users", __name__) permissions = permissions.permissions From d3530cc15f71688893fcc7c502143e7b5d6534b8 Mon Sep 17 00:00:00 2001 From: Ferdinand Thiessen Date: Thu, 18 Aug 2022 19:58:02 +0200 Subject: [PATCH 10/27] The enabled state of plugins is now loaded from database rather than config file Signed-off-by: Ferdinand Thiessen --- flaschengeist/app.py | 58 +++++++--------- flaschengeist/config.py | 11 --- flaschengeist/controller/messageController.py | 4 +- flaschengeist/controller/pluginController.py | 69 +++++++++++++++++-- flaschengeist/models/__init__.py | 2 +- flaschengeist/models/notification.py | 13 +++- flaschengeist/models/user.py | 2 + flaschengeist/plugins/__init__.py | 14 +++- flaschengeist/plugins/scheduler.py | 9 +-- flaschengeist/plugins/users/__init__.py | 2 +- 10 files changed, 120 insertions(+), 64 deletions(-) diff --git a/flaschengeist/app.py b/flaschengeist/app.py index f992005..29ece60 100644 --- a/flaschengeist/app.py +++ b/flaschengeist/app.py @@ -11,8 +11,7 @@ from werkzeug.exceptions import HTTPException from flaschengeist import logger from flaschengeist.models import Plugin from flaschengeist.utils.hook import Hook -from flaschengeist.plugins import AuthPlugin -from flaschengeist.config import config, configure_app +from flaschengeist.config import configure_app class CustomJSONEncoder(JSONEncoder): @@ -40,39 +39,30 @@ class CustomJSONEncoder(JSONEncoder): def load_plugins(app: Flask): app.config["FG_PLUGINS"] = {} - for entry_point in entry_points(group="flaschengeist.plugins"): - logger.debug(f"Found plugin: {entry_point.name} ({entry_point.dist.version})") + enabled_plugins = Plugin.query.filter(Plugin.enabled == True).all() + all_plugins = entry_points(group="flaschengeist.plugins") - if entry_point.name == config["FLASCHENGEIST"]["auth"] or ( - entry_point.name in config and config[entry_point.name].get("enabled", False) - ): - logger.debug(f"Load plugin {entry_point.name}") - try: - plugin = entry_point.load()(entry_point, config=config.get(entry_point.name, {})) - if hasattr(plugin, "blueprint") and plugin.blueprint is not None: - app.register_blueprint(plugin.blueprint) - except: - logger.error( - f"Plugin {entry_point.name} was enabled, but could not be loaded due to an error.", - exc_info=True, - ) - continue - if isinstance(plugin, AuthPlugin): - if entry_point.name != config["FLASCHENGEIST"]["auth"]: - logger.debug(f"Unload not configured AuthPlugin {entry_point.name}") - del plugin - continue - else: - logger.info(f"Using authentication plugin: {entry_point.name}") - app.config["FG_AUTH_BACKEND"] = plugin - else: - logger.info(f"Using plugin: {entry_point.name}") - app.config["FG_PLUGINS"][entry_point.name] = plugin - else: - logger.debug(f"Skip disabled plugin {entry_point.name}") - if "FG_AUTH_BACKEND" not in app.config: - logger.fatal("No authentication plugin configured or authentication plugin not found") - raise RuntimeError("No authentication plugin configured or authentication plugin not found") + for plugin in enabled_plugins: + logger.debug(f"Searching for enabled plugin {plugin.name}") + entry_point = all_plugins.select(name=plugin.name) + if not entry_point: + logger.error( + f"Plugin {plugin.name} was enabled, but could not be found.", + exc_info=True, + ) + continue + try: + loaded = entry_point[0].load()(entry_point[0]) + if hasattr(plugin, "blueprint") and plugin.blueprint is not None: + app.register_blueprint(plugin.blueprint) + except: + logger.error( + f"Plugin {plugin.name} was enabled, but could not be loaded due to an error.", + exc_info=True, + ) + continue + logger.info(f"Loaded plugin: {plugin.name}") + app.config["FG_PLUGINS"][plugin.name] = loaded def create_app(test_config=None, cli=False): diff --git a/flaschengeist/config.py b/flaschengeist/config.py index 3adae22..712d5d1 100644 --- a/flaschengeist/config.py +++ b/flaschengeist/config.py @@ -77,17 +77,6 @@ def configure_app(app, test_config=None): configure_logger() - # Always enable this builtin plugins! - update_dict( - config, - { - "auth": {"enabled": True}, - "roles": {"enabled": True}, - "users": {"enabled": True}, - "scheduler": {"enabled": True}, - }, - ) - if "secret_key" not in config["FLASCHENGEIST"]: logger.critical("No secret key was configured, please configure one for production systems!") raise RuntimeError("No secret key was configured") diff --git a/flaschengeist/controller/messageController.py b/flaschengeist/controller/messageController.py index 573a149..d9ff78c 100644 --- a/flaschengeist/controller/messageController.py +++ b/flaschengeist/controller/messageController.py @@ -1,5 +1,5 @@ -from flaschengeist.utils.hook import Hook -from flaschengeist.models import User, Role +from ..utils.hook import Hook +from ..models import User, Role class Message: diff --git a/flaschengeist/controller/pluginController.py b/flaschengeist/controller/pluginController.py index 8313935..6c16491 100644 --- a/flaschengeist/controller/pluginController.py +++ b/flaschengeist/controller/pluginController.py @@ -29,9 +29,7 @@ def get_setting(plugin_id: str, name: str, **kwargs): `KeyError` if no such setting exists in the database """ try: - setting = ( - PluginSetting.query.filter(PluginSetting.plugin == plugin_id).filter(PluginSetting.name == name).one() - ) + setting = PluginSetting.query.filter(PluginSetting.plugin == plugin_id).filter(PluginSetting.name == name).one() return setting.value except sqlalchemy.orm.exc.NoResultFound: if "default" in kwargs: @@ -49,9 +47,7 @@ def set_setting(plugin_id: str, name: str, value): value: Value to be stored """ setting = ( - PluginSetting.query.filter(PluginSetting.plugin == plugin_id) - .filter(PluginSetting.name == name) - .one_or_none() + PluginSetting.query.filter(PluginSetting.plugin == plugin_id).filter(PluginSetting.name == name).one_or_none() ) if setting is not None: if value is None: @@ -81,3 +77,64 @@ def notify(plugin_id: str, user, text: str, data=None): db.session.add(n) db.session.commit() return n.id + + +@Hook("plugins.installed") +def install_plugin(plugin_name: str): + logger.debug(f"Installing plugin {plugin_name}") + entry_point = entry_points(group="flaschengeist.plugins", name=plugin_name) + if not entry_point: + raise NotFound + + plugin = entry_point[0].load()(entry_point[0]) + entity = Plugin(name=plugin.name, version=plugin.version) + db.session.add(entity) + db.session.commit() + return entity + + +@Hook("plugin.uninstalled") +def uninstall_plugin(plugin_id: Union[str, int]): + plugin = disable_plugin(plugin_id) + logger.debug(f"Uninstall plugin {plugin.name}") + + entity = current_app.config["FG_PLUGINS"][plugin.name] + entity.uninstall() + del current_app.config["FG_PLUGINS"][plugin.name] + db.session.delete(plugin) + db.session.commit() + + +@Hook("plugins.enabled") +def enable_plugin(plugin_id: Union[str, int]): + logger.debug(f"Enabling plugin {plugin_id}") + plugin: Plugin = Plugin.query + if isinstance(plugin_id, str): + plugin = plugin.filter(Plugin.name == plugin_id).one_or_none() + if plugin is None: + logger.debug("Plugin not installed, trying to install") + plugin = install_plugin(plugin_id) + else: + plugin = plugin.get(plugin_id) + if plugin is None: + raise NotFound + plugin.enabled = True + db.session.commit() + + return plugin + + +@Hook("plugins.disabled") +def disable_plugin(plugin_id: Union[str, int]): + logger.debug(f"Disabling plugin {plugin_id}") + plugin: Plugin = Plugin.query + if isinstance(plugin_id, str): + plugin = plugin.filter(Plugin.name == plugin_id).one_or_none() + else: + plugin = plugin.get(plugin_id) + if plugin is None: + raise NotFound + plugin.enabled = False + db.session.commit() + + return plugin diff --git a/flaschengeist/models/__init__.py b/flaschengeist/models/__init__.py index 99a5e8d..096ac2e 100644 --- a/flaschengeist/models/__init__.py +++ b/flaschengeist/models/__init__.py @@ -2,4 +2,4 @@ from .session import * from .user import * from .plugin import * from .notification import * -from .image import * \ No newline at end of file +from .image import * diff --git a/flaschengeist/models/notification.py b/flaschengeist/models/notification.py index a25efd4..549a5b7 100644 --- a/flaschengeist/models/notification.py +++ b/flaschengeist/models/notification.py @@ -8,10 +8,17 @@ from ..database.types import Serial, UtcDateTime, ModelSerializeMixin class Notification(db.Model, ModelSerializeMixin): __tablename__ = "notification" id: int = db.Column("id", Serial, primary_key=True) - plugin: str = db.Column(db.String(127), nullable=False) text: str = db.Column(db.Text) data: Any = db.Column(db.PickleType(protocol=4)) time: datetime = db.Column(UtcDateTime, nullable=False, default=UtcDateTime.current_utc) - user_id_: int = db.Column("user_id", Serial, db.ForeignKey("user.id"), nullable=False) - user_: User = db.relationship("User") + user_id_: int = db.Column("user", Serial, db.ForeignKey("user.id"), nullable=False) + plugin_id_: int = db.Column("plugin", Serial, db.ForeignKey("plugin.id"), nullable=False) + user_: "User" = db.relationship("User") + plugin_: "Plugin" = db.relationship( + "Plugin", backref=db.backref("notifications_", cascade="all, delete, delete-orphan") + ) + + @property + def plugin(self): + return self.plugin_.name diff --git a/flaschengeist/models/user.py b/flaschengeist/models/user.py index c468ebf..578758c 100644 --- a/flaschengeist/models/user.py +++ b/flaschengeist/models/user.py @@ -17,11 +17,13 @@ role_permission_association_table = db.Table( db.Column("permission_id", Serial, db.ForeignKey("permission.id")), ) + class Permission(db.Model, ModelSerializeMixin): __tablename__ = "permission" name: str = db.Column(db.String(30), unique=True) _id = db.Column("id", Serial, primary_key=True) + _plugin_id: int = db.Column("plugin", Serial, db.ForeignKey("plugin.id")) class Role(db.Model, ModelSerializeMixin): diff --git a/flaschengeist/plugins/__init__.py b/flaschengeist/plugins/__init__.py index 76a35e3..f1a68a0 100644 --- a/flaschengeist/plugins/__init__.py +++ b/flaschengeist/plugins/__init__.py @@ -115,10 +115,10 @@ class BasePlugin: ``` """ - def __init__(self, entry_point: EntryPoint, config=None): + def __init__(self, entry_point: EntryPoint): """Constructor called by create_app Args: - config: Dict configuration containing the plugin section + entry_point: EntryPoint from which this plugin was loaded """ self.version = entry_point.dist.version self.name = entry_point.name @@ -127,6 +127,8 @@ class BasePlugin: def install(self): """Installation routine + Also called when updating the plugin, compare `version` and `installed_version`. + Is always called with Flask application context, it is called after the plugin permissions are installed. """ @@ -143,6 +145,14 @@ class BasePlugin: """ pass + @property + def installed_version(self): + """Installed version of the plugin""" + from ..controller import pluginController + + self.__installed_version = pluginController.get_installed_version(self.name) + return self.__installed_version + def get_setting(self, name: str, **kwargs): """Get plugin setting from database diff --git a/flaschengeist/plugins/scheduler.py b/flaschengeist/plugins/scheduler.py index a5e6eef..43a0a8b 100644 --- a/flaschengeist/plugins/scheduler.py +++ b/flaschengeist/plugins/scheduler.py @@ -2,6 +2,7 @@ from flask import Blueprint from datetime import datetime, timedelta from flaschengeist import logger +from flaschengeist.config import config from flaschengeist.plugins import BasePlugin from flaschengeist.utils.HTTP import no_content @@ -38,8 +39,8 @@ def scheduled(id: str, replace=False, **kwargs): class SchedulerPlugin(BasePlugin): - def __init__(self, entry_point, config=None): - super().__init__(entry_point, config) + def __init__(self, entry_point): + super().__init__(entry_point) self.blueprint = Blueprint(self.name, __name__) def __view_func(): @@ -52,9 +53,9 @@ class SchedulerPlugin(BasePlugin): except: logger.error("Error while executing scheduled tasks!", exc_info=True) - cron = None if config is None else config.get("cron", "passive_web").lower() + cron = config.get("scheduler", {}).get("cron", "passive_web").lower() - if cron is None or cron == "passive_web": + if cron == "passive_web": self.blueprint.teardown_app_request(__passiv_func) elif cron == "active_web": self.blueprint.add_url_rule("/cron", view_func=__view_func) diff --git a/flaschengeist/plugins/users/__init__.py b/flaschengeist/plugins/users/__init__.py index 3cd44df..7511a3f 100644 --- a/flaschengeist/plugins/users/__init__.py +++ b/flaschengeist/plugins/users/__init__.py @@ -10,7 +10,7 @@ from . import permissions from flaschengeist import logger from flaschengeist.config import config from flaschengeist.plugins import BasePlugin -from flaschengeist.models.user import User +from flaschengeist.models import User from flaschengeist.utils.decorators import login_required, extract_session, headers from flaschengeist.controller import userController from flaschengeist.utils.HTTP import created, no_content From ee38e46c12773e6da1256c27e965507a79396043 Mon Sep 17 00:00:00 2001 From: Ferdinand Thiessen Date: Thu, 18 Aug 2022 22:59:19 +0200 Subject: [PATCH 11/27] [core] Cleanup + Fix loading migrations of (dis)abled plugins Signed-off-by: Ferdinand Thiessen --- flaschengeist/alembic/__init__.py | 3 ++- flaschengeist/app.py | 7 ++---- flaschengeist/cli/install_cmd.py | 4 ++-- flaschengeist/controller/pluginController.py | 23 +++++++++++++++++++- flaschengeist/database/__init__.py | 19 +++++++++------- 5 files changed, 39 insertions(+), 17 deletions(-) diff --git a/flaschengeist/alembic/__init__.py b/flaschengeist/alembic/__init__.py index cf71018..cd7fe4a 100644 --- a/flaschengeist/alembic/__init__.py +++ b/flaschengeist/alembic/__init__.py @@ -1,4 +1,5 @@ from pathlib import Path -alembic_migrations = str(Path(__file__).resolve().parent / "migrations") +alembic_migrations_path = str(Path(__file__).resolve().parent / "migrations") +alembic_script_path = str(Path(__file__).resolve().parent) diff --git a/flaschengeist/app.py b/flaschengeist/app.py index 29ece60..ee8a6bd 100644 --- a/flaschengeist/app.py +++ b/flaschengeist/app.py @@ -5,11 +5,10 @@ from flask_cors import CORS from datetime import datetime, date from flask.json import JSONEncoder, jsonify from importlib.metadata import entry_points -from sqlalchemy.exc import OperationalError from werkzeug.exceptions import HTTPException from flaschengeist import logger -from flaschengeist.models import Plugin +from flaschengeist.controller import pluginController from flaschengeist.utils.hook import Hook from flaschengeist.config import configure_app @@ -38,11 +37,9 @@ class CustomJSONEncoder(JSONEncoder): @Hook("plugins.loaded") def load_plugins(app: Flask): app.config["FG_PLUGINS"] = {} - - enabled_plugins = Plugin.query.filter(Plugin.enabled == True).all() all_plugins = entry_points(group="flaschengeist.plugins") - for plugin in enabled_plugins: + for plugin in pluginController.get_enabled_plugins(): logger.debug(f"Searching for enabled plugin {plugin.name}") entry_point = all_plugins.select(name=plugin.name) if not entry_point: diff --git a/flaschengeist/cli/install_cmd.py b/flaschengeist/cli/install_cmd.py index 3d1b1ff..e566163 100644 --- a/flaschengeist/cli/install_cmd.py +++ b/flaschengeist/cli/install_cmd.py @@ -3,7 +3,7 @@ from click.decorators import pass_context from flask.cli import with_appcontext from flask_migrate import upgrade -from flaschengeist.alembic import alembic_migrations +from flaschengeist.alembic import alembic_migrations_path from flaschengeist.cli.plugin_cmd import install_plugin_command from flaschengeist.utils.hook import Hook @@ -14,7 +14,7 @@ from flaschengeist.utils.hook import Hook @Hook("plugins.installed") def install(ctx): # Install database - upgrade(alembic_migrations, revision="heads") + upgrade(alembic_migrations_path, revision="heads") # Install plugins install_plugin_command(ctx, [], True) diff --git a/flaschengeist/controller/pluginController.py b/flaschengeist/controller/pluginController.py index 6c16491..ba0e91b 100644 --- a/flaschengeist/controller/pluginController.py +++ b/flaschengeist/controller/pluginController.py @@ -8,14 +8,35 @@ import sqlalchemy from typing import Union from flask import current_app from werkzeug.exceptions import NotFound +from sqlalchemy.exc import OperationalError from importlib.metadata import entry_points from .. import logger from ..database import db -from ..utils import Hook +from ..utils.hook import Hook from ..models import Plugin, PluginSetting, Notification +def get_enabled_plugins(): + try: + enabled_plugins = Plugin.query.filter(Plugin.enabled == True).all() + except OperationalError as e: + + class PluginStub: + def __init__(self, name) -> None: + self.name = name + + logger.error("Could not connect to database or database not initialized! No plugins enabled!") + logger.debug("Can not query enabled plugins", exc_info=True) + enabled_plugins = [ + PluginStub("auth"), + PluginStub("roles"), + PluginStub("users"), + PluginStub("scheduler"), + ] + return enabled_plugins + + def get_setting(plugin_id: str, name: str, **kwargs): """Get plugin setting from database diff --git a/flaschengeist/database/__init__.py b/flaschengeist/database/__init__.py index 5bb30ef..21301d7 100644 --- a/flaschengeist/database/__init__.py +++ b/flaschengeist/database/__init__.py @@ -4,7 +4,9 @@ from flask_sqlalchemy import SQLAlchemy from importlib.metadata import EntryPoint, entry_points, distribution from sqlalchemy import MetaData +from flaschengeist.alembic import alembic_script_path from flaschengeist import logger +from flaschengeist.controller import pluginController # https://alembic.sqlalchemy.org/en/latest/naming.html metadata = MetaData( @@ -31,25 +33,26 @@ def configure_alembic(config: Config): uninstall can break the alembic version management. """ # Set main script location - config.set_main_option( - "script_location", str(distribution("flaschengeist").locate_file("") / "flaschengeist" / "alembic") - ) + config.set_main_option("script_location", alembic_script_path) # Set Flaschengeist's migrations migrations = [config.get_main_option("script_location") + "/migrations"] # Gather all migration paths - ep: EntryPoint - for ep in entry_points(group="flaschengeist.plugins"): + all_plugins = entry_points(group="flaschengeist.plugins") + for plugin in pluginController.get_enabled_plugins(): + entry_point = all_plugins.select(name=plugin.name) + if not entry_point: + continue try: - directory = ep.dist.locate_file("") - for loc in ep.module.split(".") + ["migrations"]: + directory = entry_point.dist.locate_file("") + for loc in entry_point.module.split(".") + ["migrations"]: directory /= loc if directory.exists(): logger.debug(f"Adding migration version path {directory}") migrations.append(str(directory.resolve())) except: - logger.warning(f"Could not load migrations of plugin {ep.name} for database migration.") + logger.warning(f"Could not load migrations of plugin {plugin.name} for database migration.") logger.debug("Plugin loading failed", exc_info=True) # write back seperator (we changed it if neither seperator nor locations were specified) From e22e38b3043de5fcbfbf86aea4742e93e4b41324 Mon Sep 17 00:00:00 2001 From: Ferdinand Thiessen Date: Mon, 22 Aug 2022 17:18:03 +0200 Subject: [PATCH 12/27] Implement custom UA parsing, allowing to update Flask and Werkzeug Signed-off-by: Ferdinand Thiessen --- flaschengeist/controller/pluginController.py | 1 + flaschengeist/controller/sessionController.py | 43 +++++++++++++++++-- flaschengeist/models/session.py | 4 +- flaschengeist/utils/decorators.py | 2 +- setup.cfg | 8 ++-- 5 files changed, 48 insertions(+), 10 deletions(-) diff --git a/flaschengeist/controller/pluginController.py b/flaschengeist/controller/pluginController.py index ba0e91b..86ea301 100644 --- a/flaschengeist/controller/pluginController.py +++ b/flaschengeist/controller/pluginController.py @@ -25,6 +25,7 @@ def get_enabled_plugins(): class PluginStub: def __init__(self, name) -> None: self.name = name + self.version = "?" logger.error("Could not connect to database or database not initialized! No plugins enabled!") logger.debug("Can not query enabled plugins", exc_info=True) diff --git a/flaschengeist/controller/sessionController.py b/flaschengeist/controller/sessionController.py index 5d5ceae..da5c81c 100644 --- a/flaschengeist/controller/sessionController.py +++ b/flaschengeist/controller/sessionController.py @@ -11,7 +11,36 @@ from ..database import db lifetime = 1800 -def validate_token(token, user_agent, permission): +def __get_user_agent_platform(ua: str): + if "Win" in ua: + return "Windows" + if "Mac" in ua: + return "Macintosh" + if "Linux" in ua: + return "Linux" + if "Android" in ua: + return "Android" + if "like Mac" in ua: + return "iOS" + return "unknown" + + +def __get_user_agent_browser(ua: str): + ua_str = ua.lower() + if "firefox" in ua_str or "fxios" in ua_str: + return "firefox" + if "safari" in ua_str: + return "safari" + if "opr/" in ua_str: + return "opera" + if "edg" in ua_str: + return "edge" + if "chrom" in ua_str or "crios" in ua_str: + return "chrome" + return "unknown" + + +def validate_token(token, request_headers, permission): """Verify session Verify a Session and Roles so if the User has permission or not. @@ -19,7 +48,7 @@ def validate_token(token, user_agent, permission): Args: token: Token to verify. - user_agent: User agent of browser to check + request_headers: Headers to validate user agent of browser permission: Permission needed to access restricted routes Returns: A Session for this given Token @@ -31,8 +60,16 @@ def validate_token(token, user_agent, permission): session = Session.query.filter_by(token=token).one_or_none() if session: logger.debug("token found, check if expired or invalid user agent differs") + + platform = request_headers.get("Sec-CH-UA-Platform", None) or __get_user_agent_platform( + request_headers.get("User-Agent", "") + ) + browser = request_headers.get("Sec-CH-UA", None) or __get_user_agent_browser( + request_headers.get("User-Agent", "") + ) + if session.expires >= datetime.now(timezone.utc) and ( - session.browser == user_agent.browser and session.platform == user_agent.platform + session.browser == browser and session.platform == platform ): if not permission or session.user_.has_permission(permission): session.refresh() diff --git a/flaschengeist/models/session.py b/flaschengeist/models/session.py index dea7c62..bf584fe 100644 --- a/flaschengeist/models/session.py +++ b/flaschengeist/models/session.py @@ -20,8 +20,8 @@ class Session(db.Model, ModelSerializeMixin): expires: datetime = db.Column(UtcDateTime) token: str = db.Column(db.String(32), unique=True) lifetime: int = db.Column(db.Integer) - browser: str = db.Column(db.String(30)) - platform: str = db.Column(db.String(30)) + browser: str = db.Column(db.String(127)) + platform: str = db.Column(db.String(64)) userid: str = "" _id = db.Column("id", Serial, primary_key=True) diff --git a/flaschengeist/utils/decorators.py b/flaschengeist/utils/decorators.py index b26f66a..34814dc 100644 --- a/flaschengeist/utils/decorators.py +++ b/flaschengeist/utils/decorators.py @@ -14,7 +14,7 @@ def extract_session(permission=None): logger.debug("Missing Authorization header or ill-formed") raise Unauthorized - session = sessionController.validate_token(token, request.user_agent, permission) + session = sessionController.validate_token(token, request.headers, permission) return session diff --git a/setup.cfg b/setup.cfg index 46433c8..4c1c786 100644 --- a/setup.cfg +++ b/setup.cfg @@ -22,14 +22,14 @@ include_package_data = True python_requires = >=3.10 packages = find: install_requires = - Flask==2.0.3 - Pillow>=9.0 + Flask>=2.2.2 + Pillow>=9.2 flask_cors flask_migrate>=3.1.0 flask_sqlalchemy>=2.5.1 - sqlalchemy>=1.4.39 + sqlalchemy>=1.4.40 toml - werkzeug==2.0.3 + werkzeug>=2.2.2 [options.extras_require] argon = argon2-cffi From 4248825af0c9a079b8c8326b3f96aba923b41e97 Mon Sep 17 00:00:00 2001 From: Ferdinand Thiessen Date: Thu, 25 Aug 2022 12:06:59 +0200 Subject: [PATCH 13/27] Revert future imports for annotations, PEP563 is still defered Signed-off-by: Ferdinand Thiessen --- flaschengeist/models/image.py | 2 ++ flaschengeist/models/notification.py | 6 ++++-- flaschengeist/models/plugin.py | 2 ++ flaschengeist/models/session.py | 7 ++++--- flaschengeist/models/user.py | 6 ++++-- 5 files changed, 16 insertions(+), 7 deletions(-) diff --git a/flaschengeist/models/image.py b/flaschengeist/models/image.py index 406fefe..101a19a 100644 --- a/flaschengeist/models/image.py +++ b/flaschengeist/models/image.py @@ -1,3 +1,5 @@ +from __future__ import annotations # TODO: Remove if python requirement is >= 3.12 (? PEP 563 is defered) + from sqlalchemy import event from pathlib import Path diff --git a/flaschengeist/models/notification.py b/flaschengeist/models/notification.py index 549a5b7..55e9640 100644 --- a/flaschengeist/models/notification.py +++ b/flaschengeist/models/notification.py @@ -1,3 +1,5 @@ +from __future__ import annotations # TODO: Remove if python requirement is >= 3.12 (? PEP 563 is defered) + from datetime import datetime from typing import Any @@ -14,8 +16,8 @@ class Notification(db.Model, ModelSerializeMixin): user_id_: int = db.Column("user", Serial, db.ForeignKey("user.id"), nullable=False) plugin_id_: int = db.Column("plugin", Serial, db.ForeignKey("plugin.id"), nullable=False) - user_: "User" = db.relationship("User") - plugin_: "Plugin" = db.relationship( + user_: User = db.relationship("User") + plugin_: Plugin = db.relationship( "Plugin", backref=db.backref("notifications_", cascade="all, delete, delete-orphan") ) diff --git a/flaschengeist/models/plugin.py b/flaschengeist/models/plugin.py index d2af46c..eb5cf35 100644 --- a/flaschengeist/models/plugin.py +++ b/flaschengeist/models/plugin.py @@ -1,3 +1,5 @@ +from __future__ import annotations # TODO: Remove if python requirement is >= 3.12 (? PEP 563 is defered) + from typing import Any from ..database import db diff --git a/flaschengeist/models/session.py b/flaschengeist/models/session.py index bf584fe..1dac1a3 100644 --- a/flaschengeist/models/session.py +++ b/flaschengeist/models/session.py @@ -1,11 +1,12 @@ +from __future__ import annotations # TODO: Remove if python requirement is >= 3.12 (? PEP 563 is defered) + from datetime import datetime, timedelta, timezone from secrets import compare_digest +from .. import logger from ..database import db from ..database.types import ModelSerializeMixin, UtcDateTime, Serial -from flaschengeist import logger - class Session(db.Model, ModelSerializeMixin): """Model for a Session @@ -26,7 +27,7 @@ class Session(db.Model, ModelSerializeMixin): _id = db.Column("id", Serial, primary_key=True) _user_id = db.Column("user_id", Serial, db.ForeignKey("user.id")) - user_: "User" = db.relationship("User", back_populates="sessions_") + user_: User = db.relationship("User", back_populates="sessions_") @property def userid(self): diff --git a/flaschengeist/models/user.py b/flaschengeist/models/user.py index 578758c..077b78c 100644 --- a/flaschengeist/models/user.py +++ b/flaschengeist/models/user.py @@ -1,3 +1,5 @@ +from __future__ import annotations # TODO: Remove if python requirement is >= 3.12 (? PEP 563 is defered) + from typing import Optional from datetime import date, datetime from sqlalchemy.orm.collections import attribute_mapped_collection @@ -62,10 +64,10 @@ class User(db.Model, ModelSerializeMixin): # Protected stuff for backend use only id_ = db.Column("id", Serial, primary_key=True) roles_: list[Role] = db.relationship("Role", secondary=association_table, cascade="save-update, merge") - sessions_: list["Session"] = db.relationship( + sessions_: list[Session] = db.relationship( "Session", back_populates="user_", cascade="all, delete, delete-orphan" ) - avatar_: Optional["Image"] = db.relationship("Image", cascade="all, delete, delete-orphan", single_parent=True) + avatar_: Optional[Image] = db.relationship("Image", cascade="all, delete, delete-orphan", single_parent=True) reset_requests_: list["_PasswordReset"] = db.relationship("_PasswordReset", cascade="all, delete, delete-orphan") # Private stuff for internal use From 973b4527dfdd3e6a96f162866cf68af75725d336 Mon Sep 17 00:00:00 2001 From: Ferdinand Thiessen Date: Thu, 25 Aug 2022 14:55:49 +0200 Subject: [PATCH 14/27] [core] UA parsing: Add backwards compatibility for platform names Signed-off-by: Ferdinand Thiessen --- flaschengeist/controller/sessionController.py | 20 ++++++++++--------- flaschengeist/plugins/auth/__init__.py | 2 +- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/flaschengeist/controller/sessionController.py b/flaschengeist/controller/sessionController.py index da5c81c..56ca32b 100644 --- a/flaschengeist/controller/sessionController.py +++ b/flaschengeist/controller/sessionController.py @@ -13,15 +13,15 @@ lifetime = 1800 def __get_user_agent_platform(ua: str): if "Win" in ua: - return "Windows" + return "windows" if "Mac" in ua: - return "Macintosh" + return "macintosh" if "Linux" in ua: - return "Linux" + return "linux" if "Android" in ua: - return "Android" + return "android" if "like Mac" in ua: - return "iOS" + return "ios" return "unknown" @@ -84,12 +84,12 @@ def validate_token(token, request_headers, permission): raise Unauthorized -def create(user, user_agent=None) -> Session: +def create(user, request_headers=None) -> Session: """Create a Session Args: user: For which User is to create a Session - user_agent: User agent to identify session + request_headers: Headers to validate user agent of browser Returns: Session: A created Token for User @@ -100,8 +100,10 @@ def create(user, user_agent=None) -> Session: token=token_str, user_=user, lifetime=lifetime, - browser=user_agent.browser, - platform=user_agent.platform, + platform=request_headers.get("Sec-CH-UA-Platform", None) + or __get_user_agent_platform(request_headers.get("User-Agent", "")), + browser=request_headers.get("Sec-CH-UA", None) + or __get_user_agent_browser(request_headers.get("User-Agent", "")), ) session.refresh() db.session.add(session) diff --git a/flaschengeist/plugins/auth/__init__.py b/flaschengeist/plugins/auth/__init__.py index 439b2a6..be20ac2 100644 --- a/flaschengeist/plugins/auth/__init__.py +++ b/flaschengeist/plugins/auth/__init__.py @@ -40,7 +40,7 @@ def login(): user = userController.login_user(userid, password) if not user: raise Unauthorized - session = sessionController.create(user, user_agent=request.user_agent) + session = sessionController.create(user, request_headers=request.headers) logger.debug(f"token is {session.token}") logger.info(f"User {userid} logged in.") From e2254b71b097a7262523683882cae20a4a7c449f Mon Sep 17 00:00:00 2001 From: Ferdinand Thiessen Date: Thu, 25 Aug 2022 15:14:11 +0200 Subject: [PATCH 15/27] [core][plugin] Unify plugin model and real plugins Plugins are now extensions of the database model, allowing plugins to access all their properties. Signed-off-by: Ferdinand Thiessen --- ...ial.py => 20482a003db8_initial_core_db.py} | 107 +++++++----- flaschengeist/app.py | 21 +-- flaschengeist/cli/install_cmd.py | 10 +- flaschengeist/cli/plugin_cmd.py | 160 ++++++++++-------- flaschengeist/controller/pluginController.py | 142 +++++++--------- flaschengeist/controller/userController.py | 96 +++++------ flaschengeist/database/types.py | 9 +- flaschengeist/models/plugin.py | 70 ++++++-- flaschengeist/models/user.py | 9 +- flaschengeist/plugins/__init__.py | 145 ++++++---------- flaschengeist/plugins/auth/__init__.py | 4 +- flaschengeist/plugins/auth_plain/__init__.py | 49 +++--- flaschengeist/plugins/message_mail.py | 4 +- flaschengeist/plugins/roles/__init__.py | 8 +- flaschengeist/plugins/scheduler.py | 9 +- flaschengeist/plugins/users/__init__.py | 8 +- 16 files changed, 434 insertions(+), 417 deletions(-) rename flaschengeist/alembic/migrations/{255b93b6beed_flaschengeist_initial.py => 20482a003db8_initial_core_db.py} (57%) diff --git a/flaschengeist/alembic/migrations/255b93b6beed_flaschengeist_initial.py b/flaschengeist/alembic/migrations/20482a003db8_initial_core_db.py similarity index 57% rename from flaschengeist/alembic/migrations/255b93b6beed_flaschengeist_initial.py rename to flaschengeist/alembic/migrations/20482a003db8_initial_core_db.py index b18a71e..a2a2445 100644 --- a/flaschengeist/alembic/migrations/255b93b6beed_flaschengeist_initial.py +++ b/flaschengeist/alembic/migrations/20482a003db8_initial_core_db.py @@ -1,8 +1,8 @@ -"""Flaschengeist: Initial +"""Initial core db -Revision ID: 255b93b6beed +Revision ID: 20482a003db8 Revises: -Create Date: 2022-02-23 14:33:02.851388 +Create Date: 2022-08-25 15:13:34.900996 """ from alembic import op @@ -11,24 +11,17 @@ import flaschengeist # revision identifiers, used by Alembic. -revision = "255b93b6beed" +revision = "20482a003db8" down_revision = None branch_labels = ("flaschengeist",) depends_on = None def upgrade(): - op.create_table( - "plugin_setting", - sa.Column("id", flaschengeist.models.Serial(), nullable=False), - sa.Column("plugin", sa.String(length=127), nullable=True), - sa.Column("name", sa.String(length=127), nullable=False), - sa.Column("value", sa.PickleType(protocol=4), nullable=True), - sa.PrimaryKeyConstraint("id", name=op.f("pk_plugin_setting")), - ) + # ### commands auto generated by Alembic - please adjust! ### op.create_table( "image", - sa.Column("id", flaschengeist.models.Serial(), nullable=False), + sa.Column("id", flaschengeist.database.types.Serial(), nullable=False), sa.Column("filename", sa.String(length=255), nullable=False), sa.Column("mimetype", sa.String(length=127), nullable=False), sa.Column("thumbnail", sa.String(length=255), nullable=True), @@ -36,27 +29,37 @@ def upgrade(): sa.PrimaryKeyConstraint("id", name=op.f("pk_image")), ) op.create_table( - "permission", - sa.Column("name", sa.String(length=30), nullable=True), - sa.Column("id", flaschengeist.models.Serial(), nullable=False), - sa.PrimaryKeyConstraint("id", name=op.f("pk_permission")), - sa.UniqueConstraint("name", name=op.f("uq_permission_name")), + "plugin", + sa.Column("id", flaschengeist.database.types.Serial(), nullable=False), + sa.Column("name", sa.String(length=127), nullable=False), + sa.Column("version", sa.String(length=30), nullable=False), + sa.Column("enabled", sa.Boolean(), nullable=True), + sa.PrimaryKeyConstraint("id", name=op.f("pk_plugin")), ) op.create_table( "role", - sa.Column("id", flaschengeist.models.Serial(), nullable=False), + sa.Column("id", flaschengeist.database.types.Serial(), nullable=False), sa.Column("name", sa.String(length=30), nullable=True), sa.PrimaryKeyConstraint("id", name=op.f("pk_role")), sa.UniqueConstraint("name", name=op.f("uq_role_name")), ) op.create_table( - "role_x_permission", - sa.Column("role_id", flaschengeist.models.Serial(), nullable=True), - sa.Column("permission_id", flaschengeist.models.Serial(), nullable=True), - sa.ForeignKeyConstraint( - ["permission_id"], ["permission.id"], name=op.f("fk_role_x_permission_permission_id_permission") - ), - sa.ForeignKeyConstraint(["role_id"], ["role.id"], name=op.f("fk_role_x_permission_role_id_role")), + "permission", + sa.Column("name", sa.String(length=30), nullable=True), + sa.Column("id", flaschengeist.database.types.Serial(), nullable=False), + sa.Column("plugin", flaschengeist.database.types.Serial(), nullable=True), + sa.ForeignKeyConstraint(["plugin"], ["plugin.id"], name=op.f("fk_permission_plugin_plugin")), + sa.PrimaryKeyConstraint("id", name=op.f("pk_permission")), + sa.UniqueConstraint("name", name=op.f("uq_permission_name")), + ) + op.create_table( + "plugin_setting", + sa.Column("id", flaschengeist.database.types.Serial(), nullable=False), + sa.Column("plugin", flaschengeist.database.types.Serial(), nullable=True), + sa.Column("name", sa.String(length=127), nullable=False), + sa.Column("value", sa.PickleType(), nullable=True), + sa.ForeignKeyConstraint(["plugin"], ["plugin.id"], name=op.f("fk_plugin_setting_plugin_plugin")), + sa.PrimaryKeyConstraint("id", name=op.f("pk_plugin_setting")), ) op.create_table( "user", @@ -67,48 +70,58 @@ def upgrade(): sa.Column("deleted", sa.Boolean(), nullable=True), sa.Column("birthday", sa.Date(), nullable=True), sa.Column("mail", sa.String(length=60), nullable=True), - sa.Column("id", flaschengeist.models.Serial(), nullable=False), - sa.Column("avatar", flaschengeist.models.Serial(), nullable=True), + sa.Column("id", flaschengeist.database.types.Serial(), nullable=False), + sa.Column("avatar", flaschengeist.database.types.Serial(), nullable=True), sa.ForeignKeyConstraint(["avatar"], ["image.id"], name=op.f("fk_user_avatar_image")), sa.PrimaryKeyConstraint("id", name=op.f("pk_user")), sa.UniqueConstraint("userid", name=op.f("uq_user_userid")), ) op.create_table( "notification", - sa.Column("id", flaschengeist.models.Serial(), nullable=False), - sa.Column("plugin", sa.String(length=127), nullable=False), + sa.Column("id", flaschengeist.database.types.Serial(), nullable=False), sa.Column("text", sa.Text(), nullable=True), sa.Column("data", sa.PickleType(), nullable=True), - sa.Column("time", flaschengeist.models.UtcDateTime(), nullable=False), - sa.Column("user_id", flaschengeist.models.Serial(), nullable=False), - sa.ForeignKeyConstraint(["user_id"], ["user.id"], name=op.f("fk_notification_user_id_user")), + sa.Column("time", flaschengeist.database.types.UtcDateTime(), nullable=False), + sa.Column("user", flaschengeist.database.types.Serial(), nullable=False), + sa.Column("plugin", flaschengeist.database.types.Serial(), nullable=False), + sa.ForeignKeyConstraint(["plugin"], ["plugin.id"], name=op.f("fk_notification_plugin_plugin")), + sa.ForeignKeyConstraint(["user"], ["user.id"], name=op.f("fk_notification_user_user")), sa.PrimaryKeyConstraint("id", name=op.f("pk_notification")), ) op.create_table( "password_reset", - sa.Column("user", flaschengeist.models.Serial(), nullable=False), + sa.Column("user", flaschengeist.database.types.Serial(), nullable=False), sa.Column("token", sa.String(length=32), nullable=True), - sa.Column("expires", flaschengeist.models.UtcDateTime(), nullable=True), + sa.Column("expires", flaschengeist.database.types.UtcDateTime(), nullable=True), sa.ForeignKeyConstraint(["user"], ["user.id"], name=op.f("fk_password_reset_user_user")), sa.PrimaryKeyConstraint("user", name=op.f("pk_password_reset")), ) + op.create_table( + "role_x_permission", + sa.Column("role_id", flaschengeist.database.types.Serial(), nullable=True), + sa.Column("permission_id", flaschengeist.database.types.Serial(), nullable=True), + sa.ForeignKeyConstraint( + ["permission_id"], ["permission.id"], name=op.f("fk_role_x_permission_permission_id_permission") + ), + sa.ForeignKeyConstraint(["role_id"], ["role.id"], name=op.f("fk_role_x_permission_role_id_role")), + ) op.create_table( "session", - sa.Column("expires", flaschengeist.models.UtcDateTime(), nullable=True), + sa.Column("expires", flaschengeist.database.types.UtcDateTime(), nullable=True), sa.Column("token", sa.String(length=32), nullable=True), sa.Column("lifetime", sa.Integer(), nullable=True), - sa.Column("browser", sa.String(length=30), nullable=True), - sa.Column("platform", sa.String(length=30), nullable=True), - sa.Column("id", flaschengeist.models.Serial(), nullable=False), - sa.Column("user_id", flaschengeist.models.Serial(), nullable=True), + sa.Column("browser", sa.String(length=127), nullable=True), + sa.Column("platform", sa.String(length=64), nullable=True), + sa.Column("id", flaschengeist.database.types.Serial(), nullable=False), + sa.Column("user_id", flaschengeist.database.types.Serial(), nullable=True), sa.ForeignKeyConstraint(["user_id"], ["user.id"], name=op.f("fk_session_user_id_user")), sa.PrimaryKeyConstraint("id", name=op.f("pk_session")), sa.UniqueConstraint("token", name=op.f("uq_session_token")), ) op.create_table( "user_attribute", - sa.Column("id", flaschengeist.models.Serial(), nullable=False), - sa.Column("user", flaschengeist.models.Serial(), nullable=False), + sa.Column("id", flaschengeist.database.types.Serial(), nullable=False), + sa.Column("user", flaschengeist.database.types.Serial(), nullable=False), sa.Column("name", sa.String(length=30), nullable=True), sa.Column("value", sa.PickleType(), nullable=True), sa.ForeignKeyConstraint(["user"], ["user.id"], name=op.f("fk_user_attribute_user_user")), @@ -116,8 +129,8 @@ def upgrade(): ) op.create_table( "user_x_role", - sa.Column("user_id", flaschengeist.models.Serial(), nullable=True), - sa.Column("role_id", flaschengeist.models.Serial(), nullable=True), + sa.Column("user_id", flaschengeist.database.types.Serial(), nullable=True), + sa.Column("role_id", flaschengeist.database.types.Serial(), nullable=True), sa.ForeignKeyConstraint(["role_id"], ["role.id"], name=op.f("fk_user_x_role_role_id_role")), sa.ForeignKeyConstraint(["user_id"], ["user.id"], name=op.f("fk_user_x_role_user_id_user")), ) @@ -129,11 +142,13 @@ def downgrade(): op.drop_table("user_x_role") op.drop_table("user_attribute") op.drop_table("session") + op.drop_table("role_x_permission") op.drop_table("password_reset") op.drop_table("notification") op.drop_table("user") - op.drop_table("role_x_permission") - op.drop_table("role") + op.drop_table("plugin_setting") op.drop_table("permission") + op.drop_table("role") + op.drop_table("plugin") op.drop_table("image") # ### end Alembic commands ### diff --git a/flaschengeist/app.py b/flaschengeist/app.py index ee8a6bd..d490965 100644 --- a/flaschengeist/app.py +++ b/flaschengeist/app.py @@ -4,7 +4,7 @@ from flask import Flask from flask_cors import CORS from datetime import datetime, date from flask.json import JSONEncoder, jsonify -from importlib.metadata import entry_points +from sqlalchemy.exc import OperationalError from werkzeug.exceptions import HTTPException from flaschengeist import logger @@ -37,21 +37,13 @@ class CustomJSONEncoder(JSONEncoder): @Hook("plugins.loaded") def load_plugins(app: Flask): app.config["FG_PLUGINS"] = {} - all_plugins = entry_points(group="flaschengeist.plugins") for plugin in pluginController.get_enabled_plugins(): logger.debug(f"Searching for enabled plugin {plugin.name}") - entry_point = all_plugins.select(name=plugin.name) - if not entry_point: - logger.error( - f"Plugin {plugin.name} was enabled, but could not be found.", - exc_info=True, - ) - continue try: - loaded = entry_point[0].load()(entry_point[0]) - if hasattr(plugin, "blueprint") and plugin.blueprint is not None: - app.register_blueprint(plugin.blueprint) + cls = plugin.entry_point.load() + if hasattr(cls, "blueprint") and cls.blueprint is not None: + app.register_blueprint(cls.blueprint) except: logger.error( f"Plugin {plugin.name} was enabled, but could not be loaded due to an error.", @@ -59,7 +51,8 @@ def load_plugins(app: Flask): ) continue logger.info(f"Loaded plugin: {plugin.name}") - app.config["FG_PLUGINS"][plugin.name] = loaded + app.config["FG_PLUGINS"][plugin.name] = cls.query.get(plugin.id) if plugin.id is not None else plugin + app.config["FG_PLUGINS"][plugin.name].load() def create_app(test_config=None, cli=False): @@ -79,7 +72,7 @@ def create_app(test_config=None, cli=False): def __get_state(): from . import __version__ as version - return jsonify({"plugins": app.config["FG_PLUGINS"], "version": version}) + return jsonify({"plugins": pluginController.get_loaded_plugins(), "version": version}) @app.errorhandler(Exception) def handle_exception(e): diff --git a/flaschengeist/cli/install_cmd.py b/flaschengeist/cli/install_cmd.py index e566163..e1e9244 100644 --- a/flaschengeist/cli/install_cmd.py +++ b/flaschengeist/cli/install_cmd.py @@ -4,7 +4,7 @@ from flask.cli import with_appcontext from flask_migrate import upgrade from flaschengeist.alembic import alembic_migrations_path -from flaschengeist.cli.plugin_cmd import install_plugin_command +from flaschengeist.controller import pluginController from flaschengeist.utils.hook import Hook @@ -12,9 +12,13 @@ from flaschengeist.utils.hook import Hook @with_appcontext @pass_context @Hook("plugins.installed") -def install(ctx): +def install(ctx: click.Context): + plugins = pluginController.get_enabled_plugins() + # Install database upgrade(alembic_migrations_path, revision="heads") # Install plugins - install_plugin_command(ctx, [], True) + for plugin in plugins: + plugin = pluginController.install_plugin(plugin.name) + pluginController.enable_plugin(plugin.id) diff --git a/flaschengeist/cli/plugin_cmd.py b/flaschengeist/cli/plugin_cmd.py index d8b90c4..5356eac 100644 --- a/flaschengeist/cli/plugin_cmd.py +++ b/flaschengeist/cli/plugin_cmd.py @@ -1,12 +1,13 @@ +import traceback import click from click.decorators import pass_context from flask import current_app from flask.cli import with_appcontext from importlib.metadata import EntryPoint, entry_points -from flaschengeist.database import db -from flaschengeist.config import config -from flaschengeist.models import Permission +from flaschengeist import logger +from flaschengeist.controller import pluginController +from werkzeug.exceptions import NotFound @click.group() @@ -14,33 +15,34 @@ def plugin(): pass -def install_plugin_command(ctx, plugin, all): - """Install one or more plugins""" - if not all and len(plugin) == 0: - ctx.fail("At least one plugin must be specified, or use `--all` flag.") - - if all: - plugins = current_app.config["FG_PLUGINS"] - else: +@plugin.command() +@click.argument("plugin", nargs=-1, required=True, type=str) +@with_appcontext +@pass_context +def enable(ctx, plugin): + """Enable one or more plugins""" + for name in plugin: + click.echo(f"Enabling {name}{'.'*(20-len(name))}", nl=False) try: - plugins = {plugin_name: current_app.config["FG_PLUGINS"][plugin_name] for plugin_name in plugin} - except KeyError as e: - ctx.fail(f"Invalid plugin name, could not find >{e.args[0]}<") + pluginController.enable_plugin(name) + click.secho(" ok", fg="green") + except NotFound: + click.secho(" not installed / not found", fg="red") - for name, plugin in plugins.items(): - click.echo(f"Installing {name}{'.'*(20-len(name))}", nl=False) - # Install permissions - if plugin.permissions: - cur_perm = set(x.name for x in Permission.query.filter(Permission.name.in_(plugin.permissions)).all()) - all_perm = set(plugin.permissions) - add = all_perm - cur_perm - if add: - db.session.bulk_save_objects([Permission(name=x) for x in all_perm]) - db.session.commit() - # Custom installation steps - plugin.install() - click.secho(" ok", fg="green") +@plugin.command() +@click.argument("plugin", nargs=-1, required=True, type=str) +@with_appcontext +@pass_context +def disable(ctx, plugin): + """Disable one or more plugins""" + for name in plugin: + click.echo(f"Disabling {name}{'.'*(20-len(name))}", nl=False) + try: + pluginController.disable_plugin(name) + click.secho(" ok", fg="green") + except NotFound: + click.secho(" not installed / not found", fg="red") @plugin.command() @@ -48,42 +50,62 @@ def install_plugin_command(ctx, plugin, all): @click.option("--all", help="Install all enabled plugins", is_flag=True) @with_appcontext @pass_context -def install(ctx, plugin, all): +def install(ctx: click.Context, plugin, all): """Install one or more plugins""" - return install_plugin_command(ctx, plugin, all) + all_plugins = entry_points(group="flaschengeist.plugins") + + if all: + plugins = [ep.name for ep in all_plugins] + elif len(plugin) > 0: + plugins = plugin + for name in plugin: + if not all_plugins.select(name=name): + ctx.fail(f"Invalid plugin name, could not find >{name}<") + else: + ctx.fail("At least one plugin must be specified, or use `--all` flag.") + + for name in plugins: + click.echo(f"Installing {name}{'.'*(20-len(name))}", nl=False) + try: + pluginController.install_plugin(name) + except Exception as e: + click.secho(" failed", fg="red") + if logger.getEffectiveLevel() > 10: + ctx.fail(f"[{e.__class__.__name__}] {e}") + else: + ctx.fail(traceback.format_exc()) + else: + click.secho(" ok", fg="green") @plugin.command() -@click.argument("plugin", nargs=-1, type=str) +@click.argument("plugin", nargs=-1, required=True, type=str) @with_appcontext @pass_context def uninstall(ctx: click.Context, plugin): """Uninstall one or more plugins""" - if len(plugin) == 0: - ctx.fail("At least one plugin must be specified") - + plugins = {plg.name: plg for plg in pluginController.get_installed_plugins() if plg.name in plugin} try: - plugins = {plugin_name: current_app.config["FG_PLUGINS"][plugin_name] for plugin_name in plugin} - except KeyError as e: - ctx.fail(f"Invalid plugin ID, could not find >{e.args[0]}<") - - if ( - click.prompt( - "You are going to uninstall:\n\n" - f"\t{', '.join([plugin_name for plugin_name in plugins.keys()])}\n\n" - "Are you sure?", - default="n", - show_choices=True, - type=click.Choice(["y", "N"], False), - ).lower() - != "y" - ): - ctx.exit() - for name, plugin in plugins.items(): - click.echo(f"Uninstalling {name}{'.'*(20-len(name))}", nl=False) - plugin.uninstall() - click.secho(" ok", fg="green") + for name in plugin: + pluginController.disable_plugin(plugins[name]) + if ( + click.prompt( + "You are going to uninstall:\n\n" + f"\t{', '.join([plugin_name for plugin_name in plugins.keys()])}\n\n" + "Are you sure?", + default="n", + show_choices=True, + type=click.Choice(["y", "N"], False), + ).lower() + != "y" + ): + ctx.exit() + click.echo(f"Uninstalling {name}{'.'*(20-len(name))}", nl=False) + pluginController.uninstall_plugin(plugins[name]) + click.secho(" ok", fg="green") + except KeyError: + ctx.fail(f"Invalid plugin ID, could not find >{name}<") @plugin.command() @@ -97,21 +119,27 @@ def ls(enabled, no_header): return p.version plugins = entry_points(group="flaschengeist.plugins") - enabled_plugins = [key for key, value in config.items() if "enabled" in value and value["enabled"]] + [ - config["FLASCHENGEIST"]["auth"] - ] + installed_plugins = {plg.name: plg for plg in pluginController.get_installed_plugins()} loaded_plugins = current_app.config["FG_PLUGINS"].keys() if not no_header: - print(f"{' '*13}{'name': <20}|{'version': >10}") - print("-" * 46) + print(f"{' '*13}{'name': <20}| version | {' ' * 8} state") + print("-" * 63) for plugin in plugins: - if enabled and plugin.name not in enabled_plugins: + is_installed = plugin.name in installed_plugins.keys() + is_enabled = is_installed and installed_plugins[plugin.name].enabled + if enabled and is_enabled: continue - print( - f"{plugin.name: <33}|{plugin_version(plugin): >12}" - f"{click.style(' (enabled)', fg='green') if plugin.name in enabled_plugins else ' (disabled)'}" - ) - - for plugin in [value for value in enabled_plugins if value not in loaded_plugins]: - print(f"{plugin: <33}|{' '*12}" f"{click.style(' (not found)', fg='red')}") + print(f"{plugin.name: <33}|{plugin_version(plugin): >12} | ", end="") + if is_enabled: + if plugin.name in loaded_plugins: + print(click.style(" enabled", fg="green")) + else: + print(click.style("(failed to load)", fg="red")) + elif is_installed: + print(click.style(" disabled", fg="yellow")) + else: + print("not installed") + for name, plugin in installed_plugins.items(): + if plugin.enabled and name not in loaded_plugins: + print(f"{name: <33}|{'': >12} |" f"{click.style(' failed to load', fg='red')}") diff --git a/flaschengeist/controller/pluginController.py b/flaschengeist/controller/pluginController.py index 86ea301..7cccc4e 100644 --- a/flaschengeist/controller/pluginController.py +++ b/flaschengeist/controller/pluginController.py @@ -3,84 +3,58 @@ Used by plugins for setting and notification functionality. """ -import sqlalchemy - from typing import Union from flask import current_app -from werkzeug.exceptions import NotFound +from werkzeug.exceptions import NotFound, BadRequest from sqlalchemy.exc import OperationalError from importlib.metadata import entry_points +from flaschengeist import version as flaschengeist_version + from .. import logger from ..database import db from ..utils.hook import Hook -from ..models import Plugin, PluginSetting, Notification +from ..plugins import Plugin, AuthPlugin +from ..models import Notification -def get_enabled_plugins(): +__required_plugins = ["users", "roles", "scheduler", "auth"] + + +def get_authentication_provider(): + return [plugin for plugin in get_loaded_plugins().values() if isinstance(plugin, AuthPlugin)] + + +def get_loaded_plugins(plugin_name: str = None): + """Get loaded plugin(s)""" + plugins = current_app.config["FG_PLUGINS"] + if plugin_name is not None: + plugins = [plugins[plugin_name]] + return {name: db.session.merge(plugins[name], load=False) for name in plugins} + + +def get_installed_plugins() -> list[Plugin]: + """Get all installed plugins""" + return Plugin.query.all() + + +def get_enabled_plugins() -> list[Plugin]: + """Get all installed and enabled plugins""" try: enabled_plugins = Plugin.query.filter(Plugin.enabled == True).all() except OperationalError as e: - - class PluginStub: - def __init__(self, name) -> None: - self.name = name - self.version = "?" - logger.error("Could not connect to database or database not initialized! No plugins enabled!") logger.debug("Can not query enabled plugins", exc_info=True) + # Fake load required plugins so the database can at least be installed enabled_plugins = [ - PluginStub("auth"), - PluginStub("roles"), - PluginStub("users"), - PluginStub("scheduler"), + entry_points(group="flaschengeist.plugins", name=name)[0].load()( + name=name, enabled=True, installed_version=flaschengeist_version + ) + for name in __required_plugins ] return enabled_plugins -def get_setting(plugin_id: str, name: str, **kwargs): - """Get plugin setting from database - - Args: - plugin_id: ID of the plugin - name: string identifying the setting - default: Default value - Returns: - Value stored in database (native python) - Raises: - `KeyError` if no such setting exists in the database - """ - try: - setting = PluginSetting.query.filter(PluginSetting.plugin == plugin_id).filter(PluginSetting.name == name).one() - return setting.value - except sqlalchemy.orm.exc.NoResultFound: - if "default" in kwargs: - return kwargs["default"] - else: - raise KeyError - - -def set_setting(plugin_id: str, name: str, value): - """Save setting in database - - Args: - plugin_id: ID of the plugin - name: String identifying the setting - value: Value to be stored - """ - setting = ( - PluginSetting.query.filter(PluginSetting.plugin == plugin_id).filter(PluginSetting.name == name).one_or_none() - ) - if setting is not None: - if value is None: - db.session.delete(setting) - else: - setting.value = value - else: - db.session.add(PluginSetting(plugin=plugin_id, name=name, value=value)) - db.session.commit() - - def notify(plugin_id: str, user, text: str, data=None): """Create a new notification for an user @@ -108,55 +82,67 @@ def install_plugin(plugin_name: str): if not entry_point: raise NotFound - plugin = entry_point[0].load()(entry_point[0]) - entity = Plugin(name=plugin.name, version=plugin.version) - db.session.add(entity) + cls = entry_point[0].load() + plugin = cls.query.filter(Plugin.name == plugin_name).one_or_none() + if plugin is None: + plugin = cls(name=plugin_name, installed_version=entry_point[0].dist.version) + db.session.add(plugin) + db.session.flush() + # Custom installation steps + plugin.install() db.session.commit() - return entity + + return plugin @Hook("plugin.uninstalled") -def uninstall_plugin(plugin_id: Union[str, int]): +def uninstall_plugin(plugin_id: Union[str, int, Plugin]): plugin = disable_plugin(plugin_id) logger.debug(f"Uninstall plugin {plugin.name}") - - entity = current_app.config["FG_PLUGINS"][plugin.name] - entity.uninstall() - del current_app.config["FG_PLUGINS"][plugin.name] + plugin.uninstall() db.session.delete(plugin) db.session.commit() @Hook("plugins.enabled") -def enable_plugin(plugin_id: Union[str, int]): +def enable_plugin(plugin_id: Union[str, int]) -> Plugin: logger.debug(f"Enabling plugin {plugin_id}") - plugin: Plugin = Plugin.query + plugin = Plugin.query if isinstance(plugin_id, str): plugin = plugin.filter(Plugin.name == plugin_id).one_or_none() - if plugin is None: - logger.debug("Plugin not installed, trying to install") - plugin = install_plugin(plugin_id) - else: + elif isinstance(plugin_id, int): plugin = plugin.get(plugin_id) - if plugin is None: - raise NotFound + else: + raise TypeError + if plugin is None: + raise NotFound plugin.enabled = True db.session.commit() - + plugin = plugin.entry_point.load().query.get(plugin.id) + current_app.config["FG_PLUGINS"][plugin.name] = plugin return plugin @Hook("plugins.disabled") -def disable_plugin(plugin_id: Union[str, int]): +def disable_plugin(plugin_id: Union[str, int, Plugin]): logger.debug(f"Disabling plugin {plugin_id}") plugin: Plugin = Plugin.query if isinstance(plugin_id, str): plugin = plugin.filter(Plugin.name == plugin_id).one_or_none() - else: + elif isinstance(plugin_id, int): plugin = plugin.get(plugin_id) + elif isinstance(plugin_id, Plugin): + plugin = plugin_id + else: + raise TypeError if plugin is None: raise NotFound + if plugin.name in __required_plugins: + raise BadRequest plugin.enabled = False db.session.commit() + if plugin.name in current_app.config["FG_PLUGINS"].keys(): + del current_app.config["FG_PLUGINS"][plugin.name] + return plugin diff --git a/flaschengeist/controller/userController.py b/flaschengeist/controller/userController.py index 610db39..520ec31 100644 --- a/flaschengeist/controller/userController.py +++ b/flaschengeist/controller/userController.py @@ -3,7 +3,6 @@ import secrets from io import BytesIO from sqlalchemy import exc -from flask import current_app from datetime import datetime, timedelta, timezone from flask.helpers import send_file from werkzeug.exceptions import NotFound, BadRequest, Forbidden @@ -16,7 +15,8 @@ from ..models.user import _PasswordReset from ..utils.hook import Hook from ..utils.datetime import from_iso_format from ..utils.foreign_keys import merge_references -from ..controller import imageController, messageController, sessionController +from ..controller import imageController, messageController, pluginController, sessionController +from ..plugins import AuthPlugin def __active_users(): @@ -41,17 +41,33 @@ def _generate_password_reset(user): return reset +def get_provider(userid: str): + return [p for p in pluginController.get_authentication_provider() if p.user_exists(userid)][0] + + +@Hook +def update_user(user: User, backend: AuthPlugin): + """Update user data from backend + + This is seperate function to provide a hook""" + backend.update_user(user) + if not user.display_name: + user.display_name = "{} {}.".format(user.firstname, user.lastname[0]) + db.session.commit() + + def login_user(username, password): logger.info("login user {{ {} }}".format(username)) - - user = find_user(username) - if not user: - logger.debug("User not found in Database.") - user = User(userid=username) - db.session.add(user) - if current_app.config["FG_AUTH_BACKEND"].login(user, password): - update_user(user) - return user + for provider in pluginController.get_authentication_provider(): + uid = provider.login(username, password) + if isinstance(uid, str): + user = get_user(uid) + if not user: + logger.debug("User not found in Database.") + user = User(userid=uid) + db.session.add(user) + update_user(user, provider) + return user return None @@ -84,14 +100,6 @@ def reset_password(token: str, password: str): db.session.commit() -@Hook -def update_user(user): - current_app.config["FG_AUTH_BACKEND"].update_user(user) - if not user.display_name: - user.display_name = "{} {}.".format(user.firstname, user.lastname[0]) - db.session.commit() - - def set_roles(user: User, roles: list[str], create=False): """Set roles of user @@ -115,7 +123,7 @@ def set_roles(user: User, roles: list[str], create=False): user.roles_ = fetched -def modify_user(user, password, new_password=None): +def modify_user(user: User, password: str, new_password: str = None): """Modify given user on the backend Args: @@ -127,7 +135,8 @@ def modify_user(user, password, new_password=None): NotImplemented: If backend is not capable of this operation BadRequest: Password is wrong or other logic issues """ - current_app.config["FG_AUTH_BACKEND"].modify_user(user, password, new_password) + provider = get_provider(user.userid) + provider.modify_user(user, password, new_password) if new_password: logger.debug(f"Password changed for user {user.userid}") @@ -165,37 +174,13 @@ def get_user(uid, deleted=False) -> User: return user -def find_user(uid_mail): - """Finding an user by userid or mail in database or auth-backend - Args: - uid_mail: userid and or mail to search for - Returns: - User if found or None - """ - mail = uid_mail.split("@") - mail = len(mail) == 2 and len(mail[0]) > 0 and len(mail[1]) > 0 - - query = User.userid == uid_mail - if mail: - query |= User.mail == uid_mail - user = User.query.filter(query).one_or_none() - if user: - update_user(user) - else: - user = current_app.config["FG_AUTH_BACKEND"].find_user(uid_mail, uid_mail if mail else None) - if user: - if not user.display_name: - user.display_name = "{} {}.".format(user.firstname, user.lastname[0]) - db.session.add(user) - db.session.commit() - return user - - @Hook def delete_user(user: User): """Delete given user""" # First let the backend delete the user, as this might fail - current_app.config["FG_AUTH_BACKEND"].delete_user(user) + provider = get_provider(user.userid) + provider.delete_user(user) + # Clear all easy relationships user.avatar_ = None user._attributes.clear() @@ -247,10 +232,14 @@ def register(data, passwd=None): set_roles(user, roles) password = passwd if passwd else secrets.token_urlsafe(16) - current_app.config["FG_AUTH_BACKEND"].create_user(user, password) try: + provider = [p for p in pluginController.get_authentication_provider() if p.can_register()][0] + provider.create_user(user, password) db.session.add(user) db.session.commit() + except IndexError: + logger.error("No authentication backend, allowing registering new users, found.") + raise BadRequest except exc.IntegrityError: raise BadRequest("userid already in use") @@ -265,7 +254,7 @@ def register(data, passwd=None): ) messageController.send_message(messageController.Message(user, text, subject)) - find_user(user.userid) + provider.update_user(user) return user @@ -274,19 +263,20 @@ def load_avatar(user: User): if user.avatar_ is not None: return imageController.send_image(image=user.avatar_) else: - avatar = current_app.config["FG_AUTH_BACKEND"].get_avatar(user) + provider = get_provider(user.userid) + avatar = provider.get_avatar(user) if len(avatar.binary) > 0: return send_file(BytesIO(avatar.binary), avatar.mimetype) raise NotFound def save_avatar(user, file): - current_app.config["FG_AUTH_BACKEND"].set_avatar(user, file) + get_provider(user.userid).set_avatar(user, file) db.session.commit() def delete_avatar(user): - current_app.config["FG_AUTH_BACKEND"].delete_avatar(user) + get_provider(user.userid).delete_avatar(user) db.session.commit() diff --git a/flaschengeist/database/types.py b/flaschengeist/database/types.py index 369bb30..645ecdd 100644 --- a/flaschengeist/database/types.py +++ b/flaschengeist/database/types.py @@ -1,4 +1,4 @@ -import sys +from importlib import import_module import datetime from sqlalchemy import BigInteger, util @@ -12,12 +12,11 @@ class ModelSerializeMixin: """ def __is_optional(self, param): - if sys.version_info < (3, 8): - return False - import typing - hint = typing.get_type_hints(self.__class__)[param] + module = import_module("flaschengeist.models").__dict__ + + hint = typing.get_type_hints(self.__class__, globalns=module)[param] if ( typing.get_origin(hint) is typing.Union and len(typing.get_args(hint)) == 2 diff --git a/flaschengeist/models/plugin.py b/flaschengeist/models/plugin.py index eb5cf35..a912afd 100644 --- a/flaschengeist/models/plugin.py +++ b/flaschengeist/models/plugin.py @@ -1,26 +1,72 @@ from __future__ import annotations # TODO: Remove if python requirement is >= 3.12 (? PEP 563 is defered) from typing import Any +from sqlalchemy.orm.collections import attribute_mapped_collection from ..database import db from ..database.types import Serial -class Plugin(db.Model): - __tablename__ = "plugin" - id: int = db.Column("id", Serial, primary_key=True) - name: str = db.Column(db.String(127), nullable=False) - version: str = db.Column(db.String(30), nullable=False) - """The latest installed version""" - enabled: bool = db.Column(db.Boolean, default=False) - - settings_ = db.relationship("PluginSetting", cascade="all, delete") - permissions_ = db.relationship("Permission", cascade="all, delete") - - class PluginSetting(db.Model): __tablename__ = "plugin_setting" id = db.Column("id", Serial, primary_key=True) plugin_id: int = db.Column("plugin", Serial, db.ForeignKey("plugin.id")) name: str = db.Column(db.String(127), nullable=False) value: Any = db.Column(db.PickleType(protocol=4)) + + +class BasePlugin(db.Model): + __tablename__ = "plugin" + id: int = db.Column("id", Serial, primary_key=True) + name: str = db.Column(db.String(127), nullable=False) + """Name of the plugin, loaded from distribution""" + installed_version: str = db.Column("version", db.String(30), nullable=False) + """The latest installed version""" + enabled: bool = db.Column(db.Boolean, default=False) + """Enabled state of the plugin""" + permissions: list = db.relationship( + "Permission", cascade="all, delete, delete-orphan", back_populates="plugin_", lazy="select" + ) + """Optional list of custom permissions used by the plugin + + A good style is to name the permissions with a prefix related to the plugin name, + to prevent clashes with other plugins. E. g. instead of *delete* use *plugin_delete*. + """ + + __settings: dict[str, "PluginSetting"] = db.relationship( + "PluginSetting", + collection_class=attribute_mapped_collection("name"), + cascade="all, delete, delete-orphan", + lazy="select", + ) + + def get_setting(self, name: str, **kwargs): + """Get plugin setting + + Args: + name: string identifying the setting + default: Default value + Returns: + Value stored in database (native python) + Raises: + `KeyError` if no such setting exists in the database + """ + try: + return self.__settings[name].value + except KeyError as e: + if "default" in kwargs: + return kwargs["default"] + raise e + + def set_setting(self, name: str, value): + """Save setting in database + + Args: + name: String identifying the setting + value: Value to be stored + """ + if value is None and name in self.__settings.keys(): + del self.__settings[name] + else: + setting = self.__settings.setdefault(name, PluginSetting(plugin_id=self.id, name=name, value=None)) + setting.value = value diff --git a/flaschengeist/models/user.py b/flaschengeist/models/user.py index 077b78c..e12db4a 100644 --- a/flaschengeist/models/user.py +++ b/flaschengeist/models/user.py @@ -24,8 +24,9 @@ class Permission(db.Model, ModelSerializeMixin): __tablename__ = "permission" name: str = db.Column(db.String(30), unique=True) - _id = db.Column("id", Serial, primary_key=True) - _plugin_id: int = db.Column("plugin", Serial, db.ForeignKey("plugin.id")) + id_ = db.Column("id", Serial, primary_key=True) + plugin_id_: int = db.Column("plugin", Serial, db.ForeignKey("plugin.id")) + plugin_ = db.relationship("Plugin", lazy="select", back_populates="permissions", enable_typechecks=False) class Role(db.Model, ModelSerializeMixin): @@ -64,9 +65,7 @@ class User(db.Model, ModelSerializeMixin): # Protected stuff for backend use only id_ = db.Column("id", Serial, primary_key=True) roles_: list[Role] = db.relationship("Role", secondary=association_table, cascade="save-update, merge") - sessions_: list[Session] = db.relationship( - "Session", back_populates="user_", cascade="all, delete, delete-orphan" - ) + sessions_: list[Session] = db.relationship("Session", back_populates="user_", cascade="all, delete, delete-orphan") avatar_: Optional[Image] = db.relationship("Image", cascade="all, delete, delete-orphan", single_parent=True) reset_requests_: list["_PasswordReset"] = db.relationship("_PasswordReset", cascade="all, delete, delete-orphan") diff --git a/flaschengeist/plugins/__init__.py b/flaschengeist/plugins/__init__.py index f1a68a0..ff53f86 100644 --- a/flaschengeist/plugins/__init__.py +++ b/flaschengeist/plugins/__init__.py @@ -4,13 +4,13 @@ """ -from typing import Optional -from importlib.metadata import Distribution, EntryPoint -from werkzeug.exceptions import MethodNotAllowed, NotFound +from typing import Union +from importlib.metadata import entry_points +from werkzeug.exceptions import NotFound from werkzeug.datastructures import FileStorage -from flaschengeist.models import User -from flaschengeist.models.user import _Avatar +from flaschengeist.models.plugin import BasePlugin +from flaschengeist.models.user import _Avatar, Permission from flaschengeist.utils.hook import HookBefore, HookAfter __all__ = [ @@ -20,7 +20,7 @@ __all__ = [ "before_role_updated", "before_update_user", "after_role_updated", - "BasePlugin", + "Plugin", "AuthPlugin", ] @@ -71,7 +71,7 @@ Passed args: """ -class BasePlugin: +class Plugin(BasePlugin): """Base class for all Plugins All plugins must derived from this class. @@ -82,47 +82,30 @@ class BasePlugin: - *models*: Your models, used for API export """ - name: str - """Name of the plugin, loaded from EntryPoint""" - - version: str - """Version of the plugin, loaded from Distribution""" - - dist: Distribution - """Distribution of this plugin""" - blueprint = None """Optional `flask.blueprint` if the plugin uses custom routes""" - permissions: list[str] = [] - """Optional list of custom permissions used by the plugin - - A good style is to name the permissions with a prefix related to the plugin name, - to prevent clashes with other plugins. E. g. instead of *delete* use *plugin_delete*. - """ - models = None """Optional module containing the SQLAlchemy models used by the plugin""" - migrations: Optional[tuple[str, str]] = None - """Optional identifiers of the migration versions - - If custom database tables are used migrations must be provided and the - head and removal versions had to be defined, e.g. - - ``` - migrations = ("head_hash", "removal_hash") - ``` - """ + @property + def version(self) -> str: + """Version of the plugin, loaded from Distribution""" + return self.dist.version - def __init__(self, entry_point: EntryPoint): - """Constructor called by create_app - Args: - entry_point: EntryPoint from which this plugin was loaded - """ - self.version = entry_point.dist.version - self.name = entry_point.name - self.dist = entry_point.dist + @property + def dist(self): + """Distribution of this plugin""" + return self.entry_point.dist + + @property + def entry_point(self): + ep = entry_points(group="flaschengeist.plugins", name=self.name) + return ep[0] + + def load(self): + """__init__ like function that is called when the plugin is initially loaded""" + pass def install(self): """Installation routine @@ -145,40 +128,6 @@ class BasePlugin: """ pass - @property - def installed_version(self): - """Installed version of the plugin""" - from ..controller import pluginController - - self.__installed_version = pluginController.get_installed_version(self.name) - return self.__installed_version - - def get_setting(self, name: str, **kwargs): - """Get plugin setting from database - - Args: - name: string identifying the setting - default: Default value - Returns: - Value stored in database (native python) - Raises: - `KeyError` if no such setting exists in the database - """ - from ..controller import pluginController - - return pluginController.get_setting(self.name, name, **kwargs) - - def set_setting(self, name: str, value): - """Save setting in database - - Args: - name: String identifying the setting - value: Value to be stored - """ - from ..controller import pluginController - - return pluginController.set_setting(self.name, name, value) - def notify(self, user, text: str, data=None): """Create a new notification for an user @@ -203,40 +152,53 @@ class BasePlugin: """ return {"version": self.version, "permissions": self.permissions} + def install_permissions(self, permissions: list[str]): + """Helper for installing a list of strings as permissions -class AuthPlugin(BasePlugin): + Args: + permissions: List of permissions to install + """ + cur_perm = set(x.name for x in self.permissions) + all_perm = set(permissions) + + new_perms = all_perm - cur_perm + self.permissions = list(filter(lambda x: x.name in permissions, self.permissions)) + [ + Permission(name=x, plugin_=self) for x in new_perms + ] + + +class AuthPlugin(Plugin): """Base class for all authentification plugins - See also `BasePlugin` + See also `Plugin` """ - def login(self, user, pw): + def login(self, login_name, password) -> Union[bool, str]: """Login routine, MUST BE IMPLEMENTED! Args: - user: User class containing at least the uid - pw: given password + login_name: The name the user entered + password: The password the user used to log in Returns: - Must return False if not found or invalid credentials, True if success + Must return False if not found or invalid credentials, otherwise the UID is returned """ raise NotImplemented - def update_user(self, user): + def update_user(self, user: "User"): """If backend is using external data, then update this user instance with external data Args: user: User object """ pass - def find_user(self, userid, mail=None): - """Find an user by userid or mail + def user_exists(self, userid) -> bool: + """Check if user exists on this backend Args: userid: Userid to search - mail: If set, mail to search Returns: - None or User + True or False """ - return None + raise NotImplemented def modify_user(self, user, password, new_password=None): """If backend is using (writeable) external data, then update the external database with the user provided. @@ -247,11 +209,14 @@ class AuthPlugin(BasePlugin): password: Password (some backends need the current password for changes) if None force edit (admin) new_password: If set a password change is requested Raises: - NotImplemented: If backend does not support this feature (or no password change) BadRequest: Logic error, e.g. password is wrong. Error: Other errors if backend went mad (are not handled and will result in a 500 error) """ - raise NotImplemented + pass + + def can_register(self): + """Check if this backend allows to register new users""" + return False def create_user(self, user, password): """If backend is using (writeable) external data, then create a new user on the external database. @@ -272,7 +237,7 @@ class AuthPlugin(BasePlugin): """ raise MethodNotAllowed - def get_avatar(self, user): + def get_avatar(self, user) -> _Avatar: """Retrieve avatar for given user (if supported by auth backend) Default behavior is to use native Image objects, diff --git a/flaschengeist/plugins/auth/__init__.py b/flaschengeist/plugins/auth/__init__.py index be20ac2..7f06302 100644 --- a/flaschengeist/plugins/auth/__init__.py +++ b/flaschengeist/plugins/auth/__init__.py @@ -6,13 +6,13 @@ from flask import Blueprint, request, jsonify from werkzeug.exceptions import Forbidden, BadRequest, Unauthorized from flaschengeist import logger -from flaschengeist.plugins import BasePlugin +from flaschengeist.plugins import Plugin from flaschengeist.utils.HTTP import no_content, created from flaschengeist.utils.decorators import login_required from flaschengeist.controller import sessionController, userController -class AuthRoutePlugin(BasePlugin): +class AuthRoutePlugin(Plugin): blueprint = Blueprint("auth", __name__) diff --git a/flaschengeist/plugins/auth_plain/__init__.py b/flaschengeist/plugins/auth_plain/__init__.py index 50ab4af..bdf01ac 100644 --- a/flaschengeist/plugins/auth_plain/__init__.py +++ b/flaschengeist/plugins/auth_plain/__init__.py @@ -7,42 +7,25 @@ import os import hashlib import binascii from werkzeug.exceptions import BadRequest -from flaschengeist.plugins import AuthPlugin, plugins_installed +from flaschengeist.plugins import AuthPlugin from flaschengeist.models import User, Role, Permission from flaschengeist.database import db from flaschengeist import logger class AuthPlain(AuthPlugin): - def install(self): - plugins_installed(self.post_install) + def can_register(self): + return True - def post_install(self, *args, **kwargs): - if User.query.filter(User.deleted == False).count() == 0: - logger.info("Installing admin user") - role = Role.query.filter(Role.name == "Superuser").first() - if role is None: - role = Role(name="Superuser", permissions=Permission.query.all()) - admin = User( - userid="admin", - firstname="Admin", - lastname="Admin", - mail="", - roles_=[role], - ) - self.modify_user(admin, None, "admin") - db.session.add(admin) - db.session.commit() - logger.warning( - "New administrator user was added, please change the password or remove it before going into" - "production mode. Initial credentials:\n" - "name: admin\n" - "password: admin" - ) - - def login(self, user: User, password: str): - if user.has_attribute("password"): - return AuthPlain._verify_password(user.get_attribute("password"), password) + def login(self, login_name, password): + users: list[User] = ( + User.query.filter((User.userid == login_name) | (User.mail == login_name)) + .filter(User._attributes.any(name="password")) + .all() + ) + for user in users: + if AuthPlain._verify_password(user.get_attribute("password"), password): + return user.userid return False def modify_user(self, user, password, new_password=None): @@ -51,6 +34,12 @@ class AuthPlain(AuthPlugin): if new_password: user.set_attribute("password", AuthPlain._hash_password(new_password)) + def user_exists(self, userid) -> bool: + return ( + db.session.query(User.id_).filter(User.userid == userid, User._attributes.any(name="password")).first() + is not None + ) + def create_user(self, user, password): if not user.userid: raise BadRequest("userid is missing for new user") @@ -68,7 +57,7 @@ class AuthPlain(AuthPlugin): return (salt + pass_hash).decode("ascii") @staticmethod - def _verify_password(stored_password, provided_password): + def _verify_password(stored_password: str, provided_password: str): salt = stored_password[:64] stored_password = stored_password[64:] pass_hash = hashlib.pbkdf2_hmac("sha3-512", provided_password.encode("utf-8"), salt.encode("ascii"), 100000) diff --git a/flaschengeist/plugins/message_mail.py b/flaschengeist/plugins/message_mail.py index 59d82d1..acc00de 100644 --- a/flaschengeist/plugins/message_mail.py +++ b/flaschengeist/plugins/message_mail.py @@ -4,13 +4,13 @@ from email.mime.multipart import MIMEMultipart from flaschengeist import logger from flaschengeist.models import User -from flaschengeist.plugins import BasePlugin +from flaschengeist.plugins import Plugin from flaschengeist.utils.hook import HookAfter from flaschengeist.controller import userController from flaschengeist.controller.messageController import Message -class MailMessagePlugin(BasePlugin): +class MailMessagePlugin(Plugin): def __init__(self, entry_point, config): super().__init__(entry_point, config) self.server = config["SERVER"] diff --git a/flaschengeist/plugins/roles/__init__.py b/flaschengeist/plugins/roles/__init__.py index cd2fae4..07380cb 100644 --- a/flaschengeist/plugins/roles/__init__.py +++ b/flaschengeist/plugins/roles/__init__.py @@ -6,7 +6,7 @@ Provides routes used to configure roles and permissions of users / roles. from werkzeug.exceptions import BadRequest from flask import Blueprint, request, jsonify -from flaschengeist.plugins import BasePlugin +from flaschengeist.plugins import Plugin from flaschengeist.controller import roleController from flaschengeist.utils.HTTP import created, no_content from flaschengeist.utils.decorators import login_required @@ -14,9 +14,11 @@ from flaschengeist.utils.decorators import login_required from . import permissions -class RolesPlugin(BasePlugin): +class RolesPlugin(Plugin): blueprint = Blueprint("roles", __name__) - permissions = permissions.permissions + + def install(self): + self.install_permissions(permissions.permissions) @RolesPlugin.blueprint.route("/roles", methods=["GET"]) diff --git a/flaschengeist/plugins/scheduler.py b/flaschengeist/plugins/scheduler.py index 43a0a8b..268bafd 100644 --- a/flaschengeist/plugins/scheduler.py +++ b/flaschengeist/plugins/scheduler.py @@ -3,7 +3,7 @@ from datetime import datetime, timedelta from flaschengeist import logger from flaschengeist.config import config -from flaschengeist.plugins import BasePlugin +from flaschengeist.plugins import Plugin from flaschengeist.utils.HTTP import no_content @@ -38,11 +38,10 @@ def scheduled(id: str, replace=False, **kwargs): return real_decorator -class SchedulerPlugin(BasePlugin): - def __init__(self, entry_point): - super().__init__(entry_point) - self.blueprint = Blueprint(self.name, __name__) +class SchedulerPlugin(Plugin): + blueprint = Blueprint("scheduler", __name__) + def load(self): def __view_func(): self.run_tasks() return no_content() diff --git a/flaschengeist/plugins/users/__init__.py b/flaschengeist/plugins/users/__init__.py index 7511a3f..e819486 100644 --- a/flaschengeist/plugins/users/__init__.py +++ b/flaschengeist/plugins/users/__init__.py @@ -9,7 +9,7 @@ from werkzeug.exceptions import BadRequest, Forbidden, MethodNotAllowed from . import permissions from flaschengeist import logger from flaschengeist.config import config -from flaschengeist.plugins import BasePlugin +from flaschengeist.plugins import Plugin from flaschengeist.models import User from flaschengeist.utils.decorators import login_required, extract_session, headers from flaschengeist.controller import userController @@ -17,9 +17,11 @@ from flaschengeist.utils.HTTP import created, no_content from flaschengeist.utils.datetime import from_iso_format -class UsersPlugin(BasePlugin): +class UsersPlugin(Plugin): blueprint = Blueprint("users", __name__) - permissions = permissions.permissions + + def install(self): + self.install_permissions(permissions.permissions) @UsersPlugin.blueprint.route("/users", methods=["POST"]) From 6ad8cd1728bcf09fe65d5aee035491b531affbac Mon Sep 17 00:00:00 2001 From: Ferdinand Thiessen Date: Thu, 25 Aug 2022 17:04:22 +0200 Subject: [PATCH 16/27] [cli] Users and roles can be now managed using the cli Signed-off-by: Ferdinand Thiessen --- flaschengeist/controller/userController.py | 4 +- flaschengeist/plugins/users/cli.py | 57 +++++++++++++++++----- setup.cfg | 3 +- 3 files changed, 48 insertions(+), 16 deletions(-) diff --git a/flaschengeist/controller/userController.py b/flaschengeist/controller/userController.py index 520ec31..65567ed 100644 --- a/flaschengeist/controller/userController.py +++ b/flaschengeist/controller/userController.py @@ -237,9 +237,9 @@ def register(data, passwd=None): provider.create_user(user, password) db.session.add(user) db.session.commit() - except IndexError: + except IndexError as e: logger.error("No authentication backend, allowing registering new users, found.") - raise BadRequest + raise e except exc.IntegrityError: raise BadRequest("userid already in use") diff --git a/flaschengeist/plugins/users/cli.py b/flaschengeist/plugins/users/cli.py index 4c9dab2..b5f6469 100644 --- a/flaschengeist/plugins/users/cli.py +++ b/flaschengeist/plugins/users/cli.py @@ -1,6 +1,8 @@ import click from flask.cli import with_appcontext -from werkzeug.exceptions import BadRequest, Conflict, NotFound +from werkzeug.exceptions import NotFound + +from flaschengeist.database import db from flaschengeist.controller import roleController, userController @@ -28,23 +30,52 @@ def user(ctx, param, value): @click.command() -@click.option("--add-role", help="Add new role", type=str) -@click.option("--set-admin", help="Make a role an admin role, adding all permissions", type=str) -@click.option("--add-user", help="Add new user interactivly", callback=user, is_flag=True, expose_value=False) +@click.option("--create", help="Add new role", is_flag=True) +@click.option("--delete", help="Delete role", is_flag=True) +@click.option("--set-admin", is_flag=True, help="Make a role an admin role, adding all permissions", type=str) +@click.argument("role", nargs=-1, required=True, type=str) +def role(create, delete, set_admin, role): + """Manage roles""" + ctx = click.get_current_context() + + if (create and delete) or (set_admin and delete): + ctx.fail("Do not mix --delete with --create or --set-admin") + + for role_name in role: + if create: + r = roleController.create_role(role_name) + else: + r = roleController.get(role_name) + if delete: + roleController.delete(r) + if set_admin: + r.permissions = roleController.get_permissions() + db.session.commit() + + +@click.command() +@click.option("--add-role", help="Add a role to an user", type=str) +@click.option("--create", help="Create new user interactivly", callback=user, is_flag=True, expose_value=False) +@click.option("--delete", help="Delete a user", is_flag=True) +@click.argument("user", nargs=-1, type=str) @with_appcontext -def users(add_role, set_admin): +def user(add_role, delete, user): + """Manage users""" from flaschengeist.database import db ctx = click.get_current_context() try: - if add_role: - roleController.create_role(add_role) - if set_admin: - role = roleController.get(set_admin) - role.permissions = roleController.get_permissions() - db.session.commit() if USER_KEY in ctx.meta: userController.register(ctx.meta[USER_KEY], ctx.meta[USER_KEY]["password"]) - except (BadRequest, NotFound) as e: - ctx.fail(e.description) + else: + for uid in user: + user = userController.get_user(uid) + if delete: + userController.delete_user(user) + elif add_role: + role = roleController.get(add_role) + user.roles_.append(role) + db.session.commit() + except NotFound: + ctx.fail(f"User not found {uid}") diff --git a/setup.cfg b/setup.cfg index 4c1c786..dcd1766 100644 --- a/setup.cfg +++ b/setup.cfg @@ -47,7 +47,8 @@ console_scripts = flaschengeist = flaschengeist.cli:main flask.commands = ldap = flaschengeist.plugins.auth_ldap.cli:ldap - users = flaschengeist.plugins.users.cli:users + user = flaschengeist.plugins.users.cli:user + role = flaschengeist.plugins.users.cli:role flaschengeist.plugins = # Authentication providers auth_plain = flaschengeist.plugins.auth_plain:AuthPlain From aa8f8f6e64ab05bd47fba7788a5b986263bac277 Mon Sep 17 00:00:00 2001 From: Ferdinand Thiessen Date: Thu, 25 Aug 2022 17:05:04 +0200 Subject: [PATCH 17/27] [core][plugin] Allow blueprints to be set on instance level This ensures blueprints are read from the plugin instance instead of the class, allowing custom routes to be added within the `load()` function. Signed-off-by: Ferdinand Thiessen --- flaschengeist/app.py | 13 ++++++++----- flaschengeist/plugins/__init__.py | 4 ++-- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/flaschengeist/app.py b/flaschengeist/app.py index d490965..be7d6fd 100644 --- a/flaschengeist/app.py +++ b/flaschengeist/app.py @@ -41,9 +41,14 @@ def load_plugins(app: Flask): for plugin in pluginController.get_enabled_plugins(): logger.debug(f"Searching for enabled plugin {plugin.name}") try: + # Load class cls = plugin.entry_point.load() - if hasattr(cls, "blueprint") and cls.blueprint is not None: - app.register_blueprint(cls.blueprint) + plugin = cls.query.get(plugin.id) if plugin.id is not None else plugin + # Custom loading tasks + plugin.load() + # Register blueprint + if hasattr(plugin, "blueprint") and plugin.blueprint is not None: + app.register_blueprint(plugin.blueprint) except: logger.error( f"Plugin {plugin.name} was enabled, but could not be loaded due to an error.", @@ -51,9 +56,7 @@ def load_plugins(app: Flask): ) continue logger.info(f"Loaded plugin: {plugin.name}") - app.config["FG_PLUGINS"][plugin.name] = cls.query.get(plugin.id) if plugin.id is not None else plugin - app.config["FG_PLUGINS"][plugin.name].load() - + app.config["FG_PLUGINS"][plugin.name] = plugin def create_app(test_config=None, cli=False): app = Flask("flaschengeist") diff --git a/flaschengeist/plugins/__init__.py b/flaschengeist/plugins/__init__.py index ff53f86..0c38c38 100644 --- a/flaschengeist/plugins/__init__.py +++ b/flaschengeist/plugins/__init__.py @@ -226,7 +226,7 @@ class AuthPlugin(Plugin): password: string """ - raise MethodNotAllowed + raise NotImplementedError def delete_user(self, user): """If backend is using (writeable) external data, then delete the user from external database. @@ -235,7 +235,7 @@ class AuthPlugin(Plugin): user: User object """ - raise MethodNotAllowed + raise NotImplementedError def get_avatar(self, user) -> _Avatar: """Retrieve avatar for given user (if supported by auth backend) From 0698327ef521ffe489cd8164068640dc738e683d Mon Sep 17 00:00:00 2001 From: Ferdinand Thiessen Date: Thu, 25 Aug 2022 17:07:12 +0200 Subject: [PATCH 18/27] [core][deps] Use sqlalchemy_utils instead of copy-paste code for merging references This fixes issues when using SQLite Signed-off-by: Ferdinand Thiessen --- flaschengeist/controller/userController.py | 2 +- flaschengeist/utils/foreign_keys.py | 51 ---------------------- setup.cfg | 1 + 3 files changed, 2 insertions(+), 52 deletions(-) delete mode 100644 flaschengeist/utils/foreign_keys.py diff --git a/flaschengeist/controller/userController.py b/flaschengeist/controller/userController.py index 65567ed..87f67c3 100644 --- a/flaschengeist/controller/userController.py +++ b/flaschengeist/controller/userController.py @@ -3,6 +3,7 @@ import secrets from io import BytesIO from sqlalchemy import exc +from sqlalchemy_utils import merge_references from datetime import datetime, timedelta, timezone from flask.helpers import send_file from werkzeug.exceptions import NotFound, BadRequest, Forbidden @@ -14,7 +15,6 @@ from ..models import Notification, User, Role from ..models.user import _PasswordReset from ..utils.hook import Hook from ..utils.datetime import from_iso_format -from ..utils.foreign_keys import merge_references from ..controller import imageController, messageController, pluginController, sessionController from ..plugins import AuthPlugin diff --git a/flaschengeist/utils/foreign_keys.py b/flaschengeist/utils/foreign_keys.py deleted file mode 100644 index ac2bedc..0000000 --- a/flaschengeist/utils/foreign_keys.py +++ /dev/null @@ -1,51 +0,0 @@ -# Borrowed from https://github.com/kvesteri/sqlalchemy-utils -# Modifications see: https://github.com/kvesteri/sqlalchemy-utils/issues/561 -# LICENSED under the BSD license, see upstream https://github.com/kvesteri/sqlalchemy-utils/blob/master/LICENSE - -import sqlalchemy as sa -from sqlalchemy.orm import object_session - - -def get_foreign_key_values(fk, obj): - mapper = sa.inspect(obj.__class__) - return dict( - ( - fk.constraint.columns.values()[index], - getattr(obj, element.column.key) - if hasattr(obj, element.column.key) - else getattr(obj, mapper.get_property_by_column(element.column).key), - ) - for index, element in enumerate(fk.constraint.elements) - ) - - -def get_referencing_foreign_keys(mixed): - tables = [mixed] - referencing_foreign_keys = set() - - for table in mixed.metadata.tables.values(): - if table not in tables: - for constraint in table.constraints: - if isinstance(constraint, sa.sql.schema.ForeignKeyConstraint): - for fk in constraint.elements: - if any(fk.references(t) for t in tables): - referencing_foreign_keys.add(fk) - return referencing_foreign_keys - - -def merge_references(from_, to, foreign_keys=None): - """ - Merge the references of an entity into another entity. - """ - if from_.__tablename__ != to.__tablename__: - raise TypeError("The tables of given arguments do not match.") - - session = object_session(from_) - foreign_keys = get_referencing_foreign_keys(from_.__table__) - - for fk in foreign_keys: - old_values = get_foreign_key_values(fk, from_) - new_values = get_foreign_key_values(fk, to) - session.query(from_.__mapper__).filter(*[k == old_values[k] for k in old_values]).update( - new_values, synchronize_session=False - ) diff --git a/setup.cfg b/setup.cfg index dcd1766..130d59a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,6 +28,7 @@ install_requires = flask_migrate>=3.1.0 flask_sqlalchemy>=2.5.1 sqlalchemy>=1.4.40 + sqlalchemy_utils>=0.38.3 toml werkzeug>=2.2.2 From 88a4dc24f297b2f1fafed765bf433e5ca028e350 Mon Sep 17 00:00:00 2001 From: Ferdinand Thiessen Date: Thu, 25 Aug 2022 18:44:21 +0200 Subject: [PATCH 19/27] [db] Fix automatic migration upgrade for plugins and core Signed-off-by: Ferdinand Thiessen --- flaschengeist/cli/install_cmd.py | 3 +-- flaschengeist/controller/pluginController.py | 12 +++++++++--- flaschengeist/database/__init__.py | 8 ++------ 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/flaschengeist/cli/install_cmd.py b/flaschengeist/cli/install_cmd.py index e1e9244..367a1bc 100644 --- a/flaschengeist/cli/install_cmd.py +++ b/flaschengeist/cli/install_cmd.py @@ -3,7 +3,6 @@ from click.decorators import pass_context from flask.cli import with_appcontext from flask_migrate import upgrade -from flaschengeist.alembic import alembic_migrations_path from flaschengeist.controller import pluginController from flaschengeist.utils.hook import Hook @@ -16,7 +15,7 @@ def install(ctx: click.Context): plugins = pluginController.get_enabled_plugins() # Install database - upgrade(alembic_migrations_path, revision="heads") + upgrade(revision="flaschengeist@head") # Install plugins for plugin in plugins: diff --git a/flaschengeist/controller/pluginController.py b/flaschengeist/controller/pluginController.py index 7cccc4e..4a84d3c 100644 --- a/flaschengeist/controller/pluginController.py +++ b/flaschengeist/controller/pluginController.py @@ -7,6 +7,7 @@ from typing import Union from flask import current_app from werkzeug.exceptions import NotFound, BadRequest from sqlalchemy.exc import OperationalError +from flask_migrate import upgrade as database_upgrade from importlib.metadata import entry_points from flaschengeist import version as flaschengeist_version @@ -83,14 +84,19 @@ def install_plugin(plugin_name: str): raise NotFound cls = entry_point[0].load() - plugin = cls.query.filter(Plugin.name == plugin_name).one_or_none() + plugin: Plugin = cls.query.filter(Plugin.name == plugin_name).one_or_none() if plugin is None: plugin = cls(name=plugin_name, installed_version=entry_point[0].dist.version) db.session.add(plugin) - db.session.flush() + db.session.commit() # Custom installation steps plugin.install() - db.session.commit() + # Check migrations + directory = entry_point[0].dist.locate_file("") + for loc in entry_point[0].module.split(".") + ["migrations"]: + directory /= loc + if directory.exists(): + database_upgrade(revision=f"{plugin_name}@head") return plugin diff --git a/flaschengeist/database/__init__.py b/flaschengeist/database/__init__.py index 21301d7..66428d0 100644 --- a/flaschengeist/database/__init__.py +++ b/flaschengeist/database/__init__.py @@ -39,11 +39,7 @@ def configure_alembic(config: Config): migrations = [config.get_main_option("script_location") + "/migrations"] # Gather all migration paths - all_plugins = entry_points(group="flaschengeist.plugins") - for plugin in pluginController.get_enabled_plugins(): - entry_point = all_plugins.select(name=plugin.name) - if not entry_point: - continue + for entry_point in entry_points(group="flaschengeist.plugins"): try: directory = entry_point.dist.locate_file("") for loc in entry_point.module.split(".") + ["migrations"]: @@ -52,7 +48,7 @@ def configure_alembic(config: Config): logger.debug(f"Adding migration version path {directory}") migrations.append(str(directory.resolve())) except: - logger.warning(f"Could not load migrations of plugin {plugin.name} for database migration.") + logger.warning(f"Could not load migrations of plugin {entry_point.name} for database migration.") logger.debug("Plugin loading failed", exc_info=True) # write back seperator (we changed it if neither seperator nor locations were specified) From 9e8117e5546f0ba616709c56de7945c08b8e5378 Mon Sep 17 00:00:00 2001 From: Ferdinand Thiessen Date: Thu, 25 Aug 2022 18:45:01 +0200 Subject: [PATCH 20/27] [plugins] Fix scheduler accessing database while unbound from session Signed-off-by: Ferdinand Thiessen --- flaschengeist/plugins/scheduler.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/flaschengeist/plugins/scheduler.py b/flaschengeist/plugins/scheduler.py index 268bafd..1a31c8a 100644 --- a/flaschengeist/plugins/scheduler.py +++ b/flaschengeist/plugins/scheduler.py @@ -60,6 +60,9 @@ class SchedulerPlugin(Plugin): self.blueprint.add_url_rule("/cron", view_func=__view_func) def run_tasks(self): + from ..database import db + self = db.session.merge(self) + changed = False now = datetime.now() status = self.get_setting("status", default=dict()) From 9f729bda6c7513bf7ed59123c9d7b4edad93ab5a Mon Sep 17 00:00:00 2001 From: Ferdinand Thiessen Date: Fri, 26 Aug 2022 17:05:03 +0200 Subject: [PATCH 21/27] [plugins] Fix `auth_ldap`, `balance`, and `pricelist` compatibility Signed-off-by: Ferdinand Thiessen --- flaschengeist/plugins/auth_ldap/__init__.py | 26 +++---- flaschengeist/plugins/balance/__init__.py | 11 +-- flaschengeist/plugins/pricelist/__init__.py | 83 ++++++++++----------- flaschengeist/plugins/pricelist/models.py | 5 +- 4 files changed, 57 insertions(+), 68 deletions(-) diff --git a/flaschengeist/plugins/auth_ldap/__init__.py b/flaschengeist/plugins/auth_ldap/__init__.py index bf2fb52..56d139b 100644 --- a/flaschengeist/plugins/auth_ldap/__init__.py +++ b/flaschengeist/plugins/auth_ldap/__init__.py @@ -11,6 +11,7 @@ from werkzeug.exceptions import BadRequest, InternalServerError, NotFound from werkzeug.datastructures import FileStorage from flaschengeist import logger +from flaschengeist.config import config from flaschengeist.controller import userController from flaschengeist.models import User, Role from flaschengeist.models.user import _Avatar @@ -18,8 +19,7 @@ from flaschengeist.plugins import AuthPlugin, before_role_updated class AuthLDAP(AuthPlugin): - def __init__(self, entry_point, config): - super().__init__(entry_point, config) + def load(self): app.config.update( LDAP_SERVER=config.get("host", "localhost"), LDAP_PORT=config.get("port", 389), @@ -54,27 +54,23 @@ class AuthLDAP(AuthPlugin): logger.debug(f"LDAP: before_role_updated called with ({role}, {new_name})") self.__modify_role(role, new_name) - def login(self, user, password): - if not user: + def login(self, login_name, password): + if not login_name: return False - return self.ldap.authenticate(user.userid, password, "uid", self.base_dn) + return self.ldap.authenticate(login_name, password, "uid", self.base_dn) - def find_user(self, userid, mail=None): - attr = self.__find(userid, mail) - if attr is not None: - user = User(userid=attr["uid"][0]) - self.__update(user, attr) - return user + def user_exists(self, userid) -> bool: + attr = self.__find(userid, None) + return attr is not None def update_user(self, user): attr = self.__find(user.userid) self.__update(user, attr) - def create_user(self, user, password): - if self.root_dn is None: - logger.error("root_dn missing in ldap config!") - raise InternalServerError + def can_register(self): + return self.root_dn is not None + def create_user(self, user, password): try: ldap_conn = self.ldap.connect(self.root_dn, self.root_secret) attributes = self.user_attributes.copy() diff --git a/flaschengeist/plugins/balance/__init__.py b/flaschengeist/plugins/balance/__init__.py index f983a9f..4ccfe20 100644 --- a/flaschengeist/plugins/balance/__init__.py +++ b/flaschengeist/plugins/balance/__init__.py @@ -4,10 +4,10 @@ Extends users plugin with balance functions """ from flask import current_app -from werkzeug.local import LocalProxy from werkzeug.exceptions import NotFound from flaschengeist import logger +from flaschengeist.config import config from flaschengeist.plugins import Plugin, plugins_loaded, before_update_user from flaschengeist.plugins.scheduler import add_scheduled @@ -56,16 +56,13 @@ def service_debit(): class BalancePlugin(Plugin): - permissions = permissions.permissions models = models - migrations = True - plugin: "BalancePlugin" = LocalProxy(lambda: current_app.config["FG_PLUGINS"][BalancePlugin.name]) + def install(self): + self.install_permissions(permissions.permissions) - def __init__(self, entry_point, config): - super(BalancePlugin, self).__init__(entry_point, config) + def load(self): from .routes import blueprint - self.blueprint = blueprint @plugins_loaded diff --git a/flaschengeist/plugins/pricelist/__init__.py b/flaschengeist/plugins/pricelist/__init__.py index 3f45351..c644ca0 100644 --- a/flaschengeist/plugins/pricelist/__init__.py +++ b/flaschengeist/plugins/pricelist/__init__.py @@ -1,39 +1,32 @@ """Pricelist plugin""" - -import pathlib -from flask import Blueprint, jsonify, request, current_app -from werkzeug.local import LocalProxy +from flask import Blueprint, jsonify, request from werkzeug.exceptions import BadRequest, Forbidden, NotFound, Unauthorized from flaschengeist import logger from flaschengeist.controller import userController from flaschengeist.controller.imageController import send_image, send_thumbnail from flaschengeist.plugins import Plugin -from flaschengeist.utils.decorators import login_required, extract_session, headers +from flaschengeist.utils.decorators import login_required, extract_session from flaschengeist.utils.HTTP import no_content from . import models from . import pricelist_controller, permissions -blueprint = Blueprint("pricelist", __name__, url_prefix="/pricelist") - - class PriceListPlugin(Plugin): - permissions = permissions.permissions - plugin = LocalProxy(lambda: current_app.config["FG_PLUGINS"][PriceListPlugin.name]) models = models + blueprint = Blueprint("pricelist", __name__, url_prefix="/pricelist") - def __init__(self, entry_point, config=None): - super().__init__(entry_point, config) - self.blueprint = blueprint - self.migrations_path = (pathlib.Path(__file__).parent / "migrations").resolve() + def install(self): + self.install_permissions(permissions.permissions) + + def load(self): config = {"discount": 0} config.update(config) -@blueprint.route("/drink-types", methods=["GET"]) -@blueprint.route("/drink-types/", methods=["GET"]) +@PriceListPlugin.blueprint.route("/drink-types", methods=["GET"]) +@PriceListPlugin.blueprint.route("/drink-types/", methods=["GET"]) def get_drink_types(identifier=None): """Get DrinkType(s) @@ -53,7 +46,7 @@ def get_drink_types(identifier=None): return jsonify(result) -@blueprint.route("/drink-types", methods=["POST"]) +@PriceListPlugin.blueprint.route("/drink-types", methods=["POST"]) @login_required(permission=permissions.CREATE_TYPE) def new_drink_type(current_session): """Create new DrinkType @@ -75,7 +68,7 @@ def new_drink_type(current_session): return jsonify(drink_type) -@blueprint.route("/drink-types/", methods=["PUT"]) +@PriceListPlugin.blueprint.route("/drink-types/", methods=["PUT"]) @login_required(permission=permissions.EDIT_TYPE) def update_drink_type(identifier, current_session): """Modify DrinkType @@ -98,7 +91,7 @@ def update_drink_type(identifier, current_session): return jsonify(drink_type) -@blueprint.route("/drink-types/", methods=["DELETE"]) +@PriceListPlugin.blueprint.route("/drink-types/", methods=["DELETE"]) @login_required(permission=permissions.DELETE_TYPE) def delete_drink_type(identifier, current_session): """Delete DrinkType @@ -116,8 +109,8 @@ def delete_drink_type(identifier, current_session): return no_content() -@blueprint.route("/tags", methods=["GET"]) -@blueprint.route("/tags/", methods=["GET"]) +@PriceListPlugin.blueprint.route("/tags", methods=["GET"]) +@PriceListPlugin.blueprint.route("/tags/", methods=["GET"]) def get_tags(identifier=None): """Get Tag(s) @@ -137,7 +130,7 @@ def get_tags(identifier=None): return jsonify(result) -@blueprint.route("/tags", methods=["POST"]) +@PriceListPlugin.blueprint.route("/tags", methods=["POST"]) @login_required(permission=permissions.CREATE_TAG) def new_tag(current_session): """Create Tag @@ -157,7 +150,7 @@ def new_tag(current_session): return jsonify(drink_type) -@blueprint.route("/tags/", methods=["PUT"]) +@PriceListPlugin.blueprint.route("/tags/", methods=["PUT"]) @login_required(permission=permissions.EDIT_TAG) def update_tag(identifier, current_session): """Modify Tag @@ -178,7 +171,7 @@ def update_tag(identifier, current_session): return jsonify(tag) -@blueprint.route("/tags/", methods=["DELETE"]) +@PriceListPlugin.blueprint.route("/tags/", methods=["DELETE"]) @login_required(permission=permissions.DELETE_TAG) def delete_tag(identifier, current_session): """Delete Tag @@ -196,8 +189,8 @@ def delete_tag(identifier, current_session): return no_content() -@blueprint.route("/drinks", methods=["GET"]) -@blueprint.route("/drinks/", methods=["GET"]) +@PriceListPlugin.blueprint.route("/drinks", methods=["GET"]) +@PriceListPlugin.blueprint.route("/drinks/", methods=["GET"]) def get_drinks(identifier=None): """Get Drink(s) @@ -253,7 +246,7 @@ def get_drinks(identifier=None): return jsonify({"drinks": drinks, "count": count}) -@blueprint.route("/list", methods=["GET"]) +@PriceListPlugin.blueprint.route("/list", methods=["GET"]) def get_pricelist(): """Get Priclist Route: ``/pricelist/list`` | Method: ``GET`` @@ -302,7 +295,7 @@ def get_pricelist(): return jsonify({"pricelist": pricelist, "count": count}) -@blueprint.route("/drinks/search/", methods=["GET"]) +@PriceListPlugin.blueprint.route("/drinks/search/", methods=["GET"]) def search_drinks(name): """Search Drink @@ -323,7 +316,7 @@ def search_drinks(name): return jsonify(pricelist_controller.get_drinks(name, public=public)) -@blueprint.route("/drinks", methods=["POST"]) +@PriceListPlugin.blueprint.route("/drinks", methods=["POST"]) @login_required(permission=permissions.CREATE) def create_drink(current_session): """Create Drink @@ -375,7 +368,7 @@ def create_drink(current_session): return jsonify(pricelist_controller.set_drink(data)) -@blueprint.route("/drinks/", methods=["PUT"]) +@PriceListPlugin.blueprint.route("/drinks/", methods=["PUT"]) @login_required(permission=permissions.EDIT) def update_drink(identifier, current_session): """Modify Drink @@ -429,7 +422,7 @@ def update_drink(identifier, current_session): return jsonify(pricelist_controller.update_drink(identifier, data)) -@blueprint.route("/drinks/", methods=["DELETE"]) +@PriceListPlugin.blueprint.route("/drinks/", methods=["DELETE"]) @login_required(permission=permissions.DELETE) def delete_drink(identifier, current_session): """Delete Drink @@ -447,7 +440,7 @@ def delete_drink(identifier, current_session): return no_content() -@blueprint.route("/prices/", methods=["DELETE"]) +@PriceListPlugin.blueprint.route("/prices/", methods=["DELETE"]) @login_required(permission=permissions.DELETE_PRICE) def delete_price(identifier, current_session): """Delete Price @@ -465,7 +458,7 @@ def delete_price(identifier, current_session): return no_content() -@blueprint.route("/volumes/", methods=["DELETE"]) +@PriceListPlugin.blueprint.route("/volumes/", methods=["DELETE"]) @login_required(permission=permissions.DELETE_VOLUME) def delete_volume(identifier, current_session): """Delete DrinkPriceVolume @@ -483,7 +476,7 @@ def delete_volume(identifier, current_session): return no_content() -@blueprint.route("/ingredients/extraIngredients", methods=["GET"]) +@PriceListPlugin.blueprint.route("/ingredients/extraIngredients", methods=["GET"]) @login_required() def get_extra_ingredients(current_session): """Get ExtraIngredients @@ -499,7 +492,7 @@ def get_extra_ingredients(current_session): return jsonify(pricelist_controller.get_extra_ingredients()) -@blueprint.route("/ingredients/", methods=["DELETE"]) +@PriceListPlugin.blueprint.route("/ingredients/", methods=["DELETE"]) @login_required(permission=permissions.DELETE_INGREDIENTS_DRINK) def delete_ingredient(identifier, current_session): """Delete Ingredient @@ -517,7 +510,7 @@ def delete_ingredient(identifier, current_session): return no_content() -@blueprint.route("/ingredients/extraIngredients", methods=["POST"]) +@PriceListPlugin.blueprint.route("/ingredients/extraIngredients", methods=["POST"]) @login_required(permission=permissions.EDIT_INGREDIENTS) def set_extra_ingredient(current_session): """Create ExtraIngredient @@ -536,7 +529,7 @@ def set_extra_ingredient(current_session): return jsonify(pricelist_controller.set_extra_ingredient(data)) -@blueprint.route("/ingredients/extraIngredients/", methods=["PUT"]) +@PriceListPlugin.blueprint.route("/ingredients/extraIngredients/", methods=["PUT"]) @login_required(permission=permissions.EDIT_INGREDIENTS) def update_extra_ingredient(identifier, current_session): """Modify ExtraIngredient @@ -556,7 +549,7 @@ def update_extra_ingredient(identifier, current_session): return jsonify(pricelist_controller.update_extra_ingredient(identifier, data)) -@blueprint.route("/ingredients/extraIngredients/", methods=["DELETE"]) +@PriceListPlugin.blueprint.route("/ingredients/extraIngredients/", methods=["DELETE"]) @login_required(permission=permissions.DELETE_INGREDIENTS) def delete_extra_ingredient(identifier, current_session): """Delete ExtraIngredient @@ -574,7 +567,7 @@ def delete_extra_ingredient(identifier, current_session): return no_content() -@blueprint.route("/settings/min_prices", methods=["GET"]) +@PriceListPlugin.blueprint.route("/settings/min_prices", methods=["GET"]) @login_required() def get_pricelist_settings_min_prices(current_session): """Get MinPrices @@ -595,7 +588,7 @@ def get_pricelist_settings_min_prices(current_session): return jsonify(min_prices) -@blueprint.route("/settings/min_prices", methods=["POST"]) +@PriceListPlugin.blueprint.route("/settings/min_prices", methods=["POST"]) @login_required(permission=permissions.EDIT_MIN_PRICES) def post_pricelist_settings_min_prices(current_session): """Create MinPrices @@ -618,7 +611,7 @@ def post_pricelist_settings_min_prices(current_session): return no_content() -@blueprint.route("/users//pricecalc_columns", methods=["GET", "PUT"]) +@PriceListPlugin.blueprint.route("/users//pricecalc_columns", methods=["GET", "PUT"]) @login_required() def get_columns(userid, current_session): """Get pricecalc_columns of an user @@ -650,7 +643,7 @@ def get_columns(userid, current_session): return no_content() -@blueprint.route("/users//pricecalc_columns_order", methods=["GET", "PUT"]) +@PriceListPlugin.blueprint.route("/users//pricecalc_columns_order", methods=["GET", "PUT"]) @login_required() def get_columns_order(userid, current_session): """Get pricecalc_columns_order of an user @@ -681,7 +674,7 @@ def get_columns_order(userid, current_session): return no_content() -@blueprint.route("/users//pricelist", methods=["GET", "PUT"]) +@PriceListPlugin.blueprint.route("/users//pricelist", methods=["GET", "PUT"]) @login_required() def get_priclist_setting(userid, current_session): """Get pricelistsetting of an user @@ -714,7 +707,7 @@ def get_priclist_setting(userid, current_session): return no_content() -@blueprint.route("/drinks//picture", methods=["POST", "DELETE"]) +@PriceListPlugin.blueprint.route("/drinks//picture", methods=["POST", "DELETE"]) @login_required(permission=permissions.EDIT) def set_picture(identifier, current_session): """Get, Create, Delete Drink Picture @@ -741,7 +734,7 @@ def set_picture(identifier, current_session): raise BadRequest -@blueprint.route("/drinks//picture", methods=["GET"]) +@PriceListPlugin.blueprint.route("/drinks//picture", methods=["GET"]) # @headers({"Cache-Control": "private, must-revalidate"}) def _get_picture(identifier): """Get Picture diff --git a/flaschengeist/plugins/pricelist/models.py b/flaschengeist/plugins/pricelist/models.py index 1d8dc23..1a5335d 100644 --- a/flaschengeist/plugins/pricelist/models.py +++ b/flaschengeist/plugins/pricelist/models.py @@ -1,8 +1,11 @@ +from __future__ import annotations # TODO: Remove if python requirement is >= 3.12 (? PEP 563 is defered) + +from typing import Optional + from flaschengeist.database import db from flaschengeist.database.types import ModelSerializeMixin, Serial from flaschengeist.models import Image -from typing import Optional drink_tag_association = db.Table( "drink_x_tag", From 7796f45097546cfd9599aba828b5df099a1bc7c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20Gr=C3=B6ger?= Date: Fri, 17 Feb 2023 15:30:05 +0100 Subject: [PATCH 22/27] feat(db) fix get plugins if no database exists --- flaschengeist/controller/pluginController.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flaschengeist/controller/pluginController.py b/flaschengeist/controller/pluginController.py index 4a84d3c..5143482 100644 --- a/flaschengeist/controller/pluginController.py +++ b/flaschengeist/controller/pluginController.py @@ -6,7 +6,7 @@ Used by plugins for setting and notification functionality. from typing import Union from flask import current_app from werkzeug.exceptions import NotFound, BadRequest -from sqlalchemy.exc import OperationalError +from sqlalchemy.exc import OperationalError, ProgrammingError from flask_migrate import upgrade as database_upgrade from importlib.metadata import entry_points @@ -43,7 +43,7 @@ def get_enabled_plugins() -> list[Plugin]: """Get all installed and enabled plugins""" try: enabled_plugins = Plugin.query.filter(Plugin.enabled == True).all() - except OperationalError as e: + except (OperationalError, ProgrammingError) as e: logger.error("Could not connect to database or database not initialized! No plugins enabled!") logger.debug("Can not query enabled plugins", exc_info=True) # Fake load required plugins so the database can at least be installed From c5436f22fa0223873ac57e0531d20f6a7a55ac2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20Gr=C3=B6ger?= Date: Fri, 17 Feb 2023 16:40:54 +0100 Subject: [PATCH 23/27] feat(ldap) fix get right config --- flaschengeist/plugins/auth_ldap/__init__.py | 34 ++++++++++++--------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/flaschengeist/plugins/auth_ldap/__init__.py b/flaschengeist/plugins/auth_ldap/__init__.py index 56d139b..2f76498 100644 --- a/flaschengeist/plugins/auth_ldap/__init__.py +++ b/flaschengeist/plugins/auth_ldap/__init__.py @@ -20,34 +20,38 @@ from flaschengeist.plugins import AuthPlugin, before_role_updated class AuthLDAP(AuthPlugin): def load(self): + self.config = config.get("auth_ldap", None) + if self.config is None: + logger.error("auth_ldap was not configured in flaschengeist.toml", exc_info=True) + raise InternalServerError app.config.update( - LDAP_SERVER=config.get("host", "localhost"), - LDAP_PORT=config.get("port", 389), - LDAP_BINDDN=config.get("bind_dn", None), - LDAP_SECRET=config.get("secret", None), - LDAP_USE_SSL=config.get("use_ssl", False), + LDAP_SERVER=self.config.get("host", "localhost"), + LDAP_PORT=self.config.get("port", 389), + LDAP_BINDDN=self.config.get("bind_dn", None), + LDAP_SECRET=self.config.get("secret", None), + LDAP_USE_SSL=self.config.get("use_ssl", False), # That's not TLS, its dirty StartTLS on unencrypted LDAP LDAP_USE_TLS=False, LDAP_TLS_VERSION=ssl.PROTOCOL_TLS, FORCE_ATTRIBUTE_VALUE_AS_LIST=True, ) if "ca_cert" in config: - app.config["LDAP_CA_CERTS_FILE"] = config["ca_cert"] + app.config["LDAP_CA_CERTS_FILE"] = self.config["ca_cert"] else: # Default is CERT_REQUIRED app.config["LDAP_REQUIRE_CERT"] = ssl.CERT_OPTIONAL self.ldap = LDAPConn(app) - self.base_dn = config["base_dn"] - self.search_dn = config.get("search_dn", "ou=people,{base_dn}").format(base_dn=self.base_dn) - self.group_dn = config.get("group_dn", "ou=group,{base_dn}").format(base_dn=self.base_dn) - self.password_hash = config.get("password_hash", "SSHA").upper() - self.object_classes = config.get("object_classes", ["inetOrgPerson"]) - self.user_attributes: dict = config.get("user_attributes", {}) - self.dn_template = config.get("dn_template") + self.base_dn = self.config["base_dn"] + self.search_dn = self.config.get("search_dn", "ou=people,{base_dn}").format(base_dn=self.base_dn) + self.group_dn = self.config.get("group_dn", "ou=group,{base_dn}").format(base_dn=self.base_dn) + self.password_hash = self.config.get("password_hash", "SSHA").upper() + self.object_classes = self.config.get("object_classes", ["inetOrgPerson"]) + self.user_attributes: dict = self.config.get("user_attributes", {}) + self.dn_template = self.config.get("dn_template") # TODO: might not be set if modify is called - self.root_dn = config.get("root_dn", None) - self.root_secret = config.get("root_secret", None) + self.root_dn = self.config.get("root_dn", None) + self.root_secret = self.config.get("root_secret", None) @before_role_updated def _role_updated(role, new_name): From e0acb80f5d9a4a3769213db3c39e52d8e6100c33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20Gr=C3=B6ger?= Date: Fri, 17 Feb 2023 17:52:20 +0100 Subject: [PATCH 24/27] feat(plugin) fix get right instance auf auth_provider --- flaschengeist/controller/pluginController.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flaschengeist/controller/pluginController.py b/flaschengeist/controller/pluginController.py index 5143482..0c0b22f 100644 --- a/flaschengeist/controller/pluginController.py +++ b/flaschengeist/controller/pluginController.py @@ -23,7 +23,7 @@ __required_plugins = ["users", "roles", "scheduler", "auth"] def get_authentication_provider(): - return [plugin for plugin in get_loaded_plugins().values() if isinstance(plugin, AuthPlugin)] + return [current_app.config["FG_PLUGINS"][plugin.name] for plugin in get_loaded_plugins().values() if isinstance(plugin, AuthPlugin)] def get_loaded_plugins(plugin_name: str = None): From a50ba403fc3040f141c552d74ee92b8494f3d179 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20Gr=C3=B6ger?= Date: Fri, 17 Feb 2023 20:40:27 +0100 Subject: [PATCH 25/27] feat(ldap) fix sync from ldap --- flaschengeist/plugins/auth_ldap/cli.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/flaschengeist/plugins/auth_ldap/cli.py b/flaschengeist/plugins/auth_ldap/cli.py index d306d21..e3d772f 100644 --- a/flaschengeist/plugins/auth_ldap/cli.py +++ b/flaschengeist/plugins/auth_ldap/cli.py @@ -1,6 +1,7 @@ import click from flask import current_app from flask.cli import with_appcontext +from werkzeug.exceptions import NotFound @click.command(no_args_is_help=True) @@ -13,8 +14,10 @@ def ldap(ctx, sync): from flaschengeist.controller import userController from flaschengeist.plugins.auth_ldap import AuthLDAP from ldap3 import SUBTREE + from flaschengeist.models import User + from flaschengeist.database import db - auth_ldap: AuthLDAP = current_app.config.get("FG_AUTH_BACKEND") + auth_ldap: AuthLDAP = current_app.config.get("FG_PLUGINS").get("auth_ldap") if auth_ldap is None or not isinstance(auth_ldap, AuthLDAP): ctx.fail("auth_ldap plugin not found or not enabled!") conn = auth_ldap.ldap.connection @@ -24,4 +27,9 @@ def ldap(ctx, sync): ldap_users_response = conn.response for ldap_user in ldap_users_response: uid = ldap_user["attributes"]["uid"][0] - userController.find_user(uid) + try: + user = userController.get_user(uid) + except NotFound: + user = User(userid=uid) + db.session.add(user) + userController.update_user(user, auth_ldap) From d475f3f8e2a1950b39d365c756a42f1024bc407f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20Gr=C3=B6ger?= Date: Sat, 18 Feb 2023 15:11:42 +0100 Subject: [PATCH 26/27] feat(ldap) fix login on ldap --- flaschengeist/plugins/auth_ldap/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/flaschengeist/plugins/auth_ldap/__init__.py b/flaschengeist/plugins/auth_ldap/__init__.py index 2f76498..8a99284 100644 --- a/flaschengeist/plugins/auth_ldap/__init__.py +++ b/flaschengeist/plugins/auth_ldap/__init__.py @@ -61,7 +61,7 @@ class AuthLDAP(AuthPlugin): def login(self, login_name, password): if not login_name: return False - return self.ldap.authenticate(login_name, password, "uid", self.base_dn) + return login_name if self.ldap.authenticate(login_name, password, "uid", self.base_dn) else False def user_exists(self, userid) -> bool: attr = self.__find(userid, None) @@ -306,3 +306,5 @@ class AuthLDAP(AuthPlugin): except (LDAPPasswordIsMandatoryError, LDAPBindError): raise BadRequest + except IndexError as e: + logger.error("Roles in LDAP", exc_info=True) From ba93345a0938c94751a8a394d9db0ae1630e1bd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20Gr=C3=B6ger?= Date: Sat, 18 Feb 2023 15:48:53 +0100 Subject: [PATCH 27/27] feat(users) fix cli if user get role, that provider is updatet too --- flaschengeist/plugins/users/cli.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flaschengeist/plugins/users/cli.py b/flaschengeist/plugins/users/cli.py index b5f6469..5e69d91 100644 --- a/flaschengeist/plugins/users/cli.py +++ b/flaschengeist/plugins/users/cli.py @@ -1,4 +1,5 @@ import click +import sqlalchemy.exc from flask.cli import with_appcontext from werkzeug.exceptions import NotFound @@ -76,6 +77,7 @@ def user(add_role, delete, user): elif add_role: role = roleController.get(add_role) user.roles_.append(role) + userController.modify_user(user, None) db.session.commit() except NotFound: ctx.fail(f"User not found {uid}")