From 507eb8d711ad1bc4283684ec02974f38956970f8 Mon Sep 17 00:00:00 2001 From: Ferdinand Thiessen Date: Wed, 23 Feb 2022 15:12:45 +0100 Subject: [PATCH] fix(migrations): Fix rebase issues --- flaschengeist/alembic/__init__.py | 0 .../alembic}/alembic.ini | 3 +- {migrations => flaschengeist/alembic}/env.py | 3 +- .../255b93b6beed_flaschengeist_initial.py | 19 ++----- flaschengeist/alembic/migrations/__init__.py | 0 .../alembic}/script.py.mako | 0 flaschengeist/cli/__init__.py | 4 +- flaschengeist/cli/plugin_cmd.py | 33 ++++++++---- flaschengeist/database.py | 51 +++++++++++-------- flaschengeist/plugins/__init__.py | 15 ++++-- flaschengeist/plugins/balance/__init__.py | 9 ++-- ...ion.py => 98f2733bbe45_balance_initial.py} | 10 ++-- ...n.py => 58ab9b6a8839_pricelist_initial.py} | 10 ++-- flaschengeist/utils/plugin.py | 49 ------------------ setup.cfg | 2 +- 15 files changed, 92 insertions(+), 116 deletions(-) create mode 100644 flaschengeist/alembic/__init__.py rename {migrations => flaschengeist/alembic}/alembic.ini (92%) rename {migrations => flaschengeist/alembic}/env.py (94%) rename migrations/versions/d3026757c7cb_initial_migration.py => flaschengeist/alembic/migrations/255b93b6beed_flaschengeist_initial.py (91%) create mode 100644 flaschengeist/alembic/migrations/__init__.py rename {migrations => flaschengeist/alembic}/script.py.mako (100%) rename flaschengeist/plugins/balance/migrations/{f07df84f7a95_initial_balance_migration.py => 98f2733bbe45_balance_initial.py} (91%) rename flaschengeist/plugins/pricelist/migrations/{7d9d306be676_initial_pricelist_migration.py => 58ab9b6a8839_pricelist_initial.py} (97%) delete mode 100644 flaschengeist/utils/plugin.py diff --git a/flaschengeist/alembic/__init__.py b/flaschengeist/alembic/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/migrations/alembic.ini b/flaschengeist/alembic/alembic.ini similarity index 92% rename from migrations/alembic.ini rename to flaschengeist/alembic/alembic.ini index e1023e2..f9e9d8e 100644 --- a/migrations/alembic.ini +++ b/flaschengeist/alembic/alembic.ini @@ -1,4 +1,5 @@ # A generic, single database configuration. +# No used by flaschengeist [alembic] # template used to generate migration files @@ -9,7 +10,7 @@ # revision_environment = false version_path_separator = os -version_locations = %(here)s/versions +version_locations = %(here)s/migrations # Logging configuration [loggers] diff --git a/migrations/env.py b/flaschengeist/alembic/env.py similarity index 94% rename from migrations/env.py rename to flaschengeist/alembic/env.py index f2bf297..f8e05d5 100644 --- a/migrations/env.py +++ b/flaschengeist/alembic/env.py @@ -1,5 +1,6 @@ import logging from logging.config import fileConfig +from pathlib import Path from flask import current_app from alembic import context @@ -9,7 +10,7 @@ config = context.config # Interpret the config file for Python logging. # This line sets up loggers basically. -fileConfig(config.config_file_name) +fileConfig(Path(config.get_main_option("script_location")) / config.config_file_name.split("/")[-1]) logger = logging.getLogger("alembic.env") config.set_main_option("sqlalchemy.url", str(current_app.extensions["migrate"].db.get_engine().url).replace("%", "%%")) diff --git a/migrations/versions/d3026757c7cb_initial_migration.py b/flaschengeist/alembic/migrations/255b93b6beed_flaschengeist_initial.py similarity index 91% rename from migrations/versions/d3026757c7cb_initial_migration.py rename to flaschengeist/alembic/migrations/255b93b6beed_flaschengeist_initial.py index 23f55b5..b7deac6 100644 --- a/migrations/versions/d3026757c7cb_initial_migration.py +++ b/flaschengeist/alembic/migrations/255b93b6beed_flaschengeist_initial.py @@ -1,8 +1,8 @@ -"""Initial migration. +"""Flaschengeist: Initial -Revision ID: d3026757c7cb +Revision ID: 255b93b6beed Revises: -Create Date: 2021-12-19 20:34:34.122576 +Create Date: 2022-02-23 14:33:02.851388 """ from alembic import op @@ -11,9 +11,9 @@ import flaschengeist # revision identifiers, used by Alembic. -revision = "d3026757c7cb" +revision = "255b93b6beed" down_revision = None -branch_labels = None +branch_labels = ("flaschengeist",) depends_on = None @@ -35,14 +35,6 @@ def upgrade(): sa.PrimaryKeyConstraint("id", name=op.f("pk_permission")), sa.UniqueConstraint("name", name=op.f("uq_permission_name")), ) - op.create_table( - "plugin_setting", - sa.Column("id", flaschengeist.models.Serial(), nullable=False), - sa.Column("plugin", sa.String(length=30), nullable=True), - sa.Column("name", sa.String(length=30), nullable=False), - sa.Column("value", sa.PickleType(), nullable=True), - sa.PrimaryKeyConstraint("id", name=op.f("pk_plugin_setting")), - ) op.create_table( "role", sa.Column("id", flaschengeist.models.Serial(), nullable=False), @@ -135,7 +127,6 @@ def downgrade(): op.drop_table("user") op.drop_table("role_x_permission") op.drop_table("role") - op.drop_table("plugin_setting") op.drop_table("permission") op.drop_table("image") # ### end Alembic commands ### diff --git a/flaschengeist/alembic/migrations/__init__.py b/flaschengeist/alembic/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/migrations/script.py.mako b/flaschengeist/alembic/script.py.mako similarity index 100% rename from migrations/script.py.mako rename to flaschengeist/alembic/script.py.mako diff --git a/flaschengeist/cli/__init__.py b/flaschengeist/cli/__init__.py index e86e60c..ed93b5d 100644 --- a/flaschengeist/cli/__init__.py +++ b/flaschengeist/cli/__init__.py @@ -84,7 +84,7 @@ def cli(): def main(*args, **kwargs): - # from .plugin_cmd import plugin + from .plugin_cmd import plugin from .export_cmd import export from .docs_cmd import docs from .run_cmd import run @@ -92,8 +92,8 @@ def main(*args, **kwargs): # Override logging level environ.setdefault("FG_LOGGING", logging.getLevelName(LOGGING_MAX)) - # cli.add_command(plugin) cli.add_command(export) cli.add_command(docs) + cli.add_command(plugin) cli.add_command(run) cli(*args, **kwargs) diff --git a/flaschengeist/cli/plugin_cmd.py b/flaschengeist/cli/plugin_cmd.py index 3cad5f7..bb0dd98 100644 --- a/flaschengeist/cli/plugin_cmd.py +++ b/flaschengeist/cli/plugin_cmd.py @@ -2,7 +2,9 @@ import click from click.decorators import pass_context from flask import current_app from flask.cli import with_appcontext -from flaschengeist.utils.plugin import get_plugins, plugin_version +from importlib_metadata import EntryPoint, entry_points + +from flaschengeist.config import config @click.group() @@ -67,17 +69,28 @@ def uninstall(ctx: click.Context, plugin): @plugin.command() @click.option("--enabled", "-e", help="List only enabled plugins", is_flag=True) +@click.option("--no-header", "-n", help="Do not show header", is_flag=True) @with_appcontext -def ls(enabled): - if enabled: - plugins = current_app.config["FG_PLUGINS"].values() - else: - plugins = get_plugins() +def ls(enabled, no_header): + def plugin_version(p): + if isinstance(p, EntryPoint): + return p.dist.version + return p.version - print(f"{' '*13}{'name': <20}|{'version': >10}") - print("-" * 46) + plugins = entry_points(group="flaschengeist.plugins") + enabled_plugins = [key for key, value in config.items() if "enabled" in value] + [config["FLASCHENGEIST"]["auth"]] + loaded_plugins = current_app.config["FG_PLUGINS"].keys() + + if not no_header: + print(f"{' '*13}{'name': <20}|{'version': >10}") + print("-" * 46) for plugin in plugins: + if enabled and plugin.name not in enabled_plugins: + continue print( - f"{plugin.id: <33}|{plugin_version(plugin): >12}" - f"{click.style(' (enabled)', fg='green') if plugin.id in current_app.config['FG_PLUGINS'] else click.style(' (disabled)', fg='red')}" + f"{plugin.name: <33}|{plugin_version(plugin): >12}" + f"{click.style(' (enabled)', fg='green') if plugin.name in enabled_plugins else ' (disabled)'}" ) + + for plugin in [value for value in enabled_plugins if value not in loaded_plugins]: + print(f"{plugin: <33}|{' '*12}" f"{click.style(' (not found)', fg='red')}") diff --git a/flaschengeist/database.py b/flaschengeist/database.py index ccafd8d..a2c4672 100644 --- a/flaschengeist/database.py +++ b/flaschengeist/database.py @@ -1,9 +1,11 @@ import os -from flask import current_app -from flask_migrate import Migrate +from flask_migrate import Migrate, Config from flask_sqlalchemy import SQLAlchemy +from importlib_metadata import EntryPoint from sqlalchemy import MetaData +from flaschengeist import logger + # https://alembic.sqlalchemy.org/en/latest/naming.html metadata = MetaData( naming_convention={ @@ -21,31 +23,40 @@ migrate = Migrate() @migrate.configure -def configure_alembic(config): +def configure_alembic(config: Config): """Alembic configuration hook Inject all migrations paths into the ``version_locations`` config option. This includes even disabled plugins, as simply disabling a plugin without uninstall can break the alembic version management. """ - import inspect, pathlib - from flaschengeist.utils.plugin import get_plugins + from importlib_metadata import entry_points, distribution - # Load migration paths from plugins - migrations = [(pathlib.Path(inspect.getfile(p)).parent / "migrations") for p in get_plugins()] - migrations = [str(m.resolve()) for m in migrations if m.exists()] - if len(migrations) > 0: - # Get configured paths - paths = config.get_main_option("version_locations") - # Get configured path seperator - sep = config.get_main_option("version_path_separator", "os") - if paths: - # Insert configured paths at the front, before plugin migrations - migrations.insert(0, config.get_main_option("version_locations")) - sep = os.pathsep if sep == "os" else " " if sep == "space" else sep - # write back seperator (we changed it if neither seperator nor locations were specified) - config.set_main_option("version_path_separator", sep) - config.set_main_option("version_locations", sep.join(migrations)) + # Set main script location + config.set_main_option( + "script_location", str(distribution("flaschengeist").locate_file("") / "flaschengeist" / "alembic") + ) + + # Set Flaschengeist's migrations + migrations = [config.get_main_option("script_location") + "/migrations"] + + # Gather all migration paths + ep: EntryPoint + for ep in entry_points(group="flaschengeist.plugins"): + try: + directory = ep.dist.locate_file("") + for loc in ep.module.split(".") + ["migrations"]: + directory /= loc + if directory.exists(): + logger.debug(f"Adding migration version path {directory}") + migrations.append(str(directory.resolve())) + except: + logger.warning(f"Could not load migrations of plugin {ep.name} for database migration.") + logger.debug("Plugin loading failed", exc_info=True) + + # write back seperator (we changed it if neither seperator nor locations were specified) + config.set_main_option("version_path_separator", os.pathsep) + config.set_main_option("version_locations", os.pathsep.join(set(migrations))) return config diff --git a/flaschengeist/plugins/__init__.py b/flaschengeist/plugins/__init__.py index 41661e9..d337d6c 100644 --- a/flaschengeist/plugins/__init__.py +++ b/flaschengeist/plugins/__init__.py @@ -4,6 +4,7 @@ """ +from typing import Optional from importlib_metadata import Distribution, EntryPoint from werkzeug.exceptions import MethodNotAllowed, NotFound from werkzeug.datastructures import FileStorage @@ -102,8 +103,16 @@ class Plugin: models = None """Optional module containing the SQLAlchemy models used by the plugin""" - migrations_path = None - """Optional location of the path to migration files, required if custome db tables are used""" + migrations: Optional[tuple[str, str]] = None + """Optional identifiers of the migration versions + + If custom database tables are used migrations must be provided and the + head and removal versions had to be defined, e.g. + + ``` + migrations = ("head_hash", "removal_hash") + ``` + """ def __init__(self, entry_point: EntryPoint, config=None): """Constructor called by create_app @@ -145,7 +154,7 @@ class Plugin: """ from ..controller import pluginController - return pluginController.get_setting(self.id, name, **kwargs) + return pluginController.get_setting(self.name, name, **kwargs) def set_setting(self, name: str, value): """Save setting in database diff --git a/flaschengeist/plugins/balance/__init__.py b/flaschengeist/plugins/balance/__init__.py index 3678ded..f983a9f 100644 --- a/flaschengeist/plugins/balance/__init__.py +++ b/flaschengeist/plugins/balance/__init__.py @@ -3,8 +3,7 @@ Extends users plugin with balance functions """ -import pathlib -from flask import Blueprint, current_app +from flask import current_app from werkzeug.local import LocalProxy from werkzeug.exceptions import NotFound @@ -58,8 +57,10 @@ def service_debit(): class BalancePlugin(Plugin): permissions = permissions.permissions - plugin: "BalancePlugin" = LocalProxy(lambda: current_app.config["FG_PLUGINS"][BalancePlugin.name]) models = models + migrations = True + + plugin: "BalancePlugin" = LocalProxy(lambda: current_app.config["FG_PLUGINS"][BalancePlugin.name]) def __init__(self, entry_point, config): super(BalancePlugin, self).__init__(entry_point, config) @@ -67,8 +68,6 @@ class BalancePlugin(Plugin): self.blueprint = blueprint - self.migrations_path = (pathlib.Path(__file__).parent / "migrations").resolve() - @plugins_loaded def post_loaded(*args, **kwargs): if config.get("allow_service_debit", False) and "events" in current_app.config["FG_PLUGINS"]: diff --git a/flaschengeist/plugins/balance/migrations/f07df84f7a95_initial_balance_migration.py b/flaschengeist/plugins/balance/migrations/98f2733bbe45_balance_initial.py similarity index 91% rename from flaschengeist/plugins/balance/migrations/f07df84f7a95_initial_balance_migration.py rename to flaschengeist/plugins/balance/migrations/98f2733bbe45_balance_initial.py index ecde05e..2a9d322 100644 --- a/flaschengeist/plugins/balance/migrations/f07df84f7a95_initial_balance_migration.py +++ b/flaschengeist/plugins/balance/migrations/98f2733bbe45_balance_initial.py @@ -1,8 +1,8 @@ -"""Initial balance migration +"""balance: initial -Revision ID: f07df84f7a95 +Revision ID: 98f2733bbe45 Revises: -Create Date: 2021-12-19 21:12:53.192267 +Create Date: 2022-02-23 14:41:03.089145 """ from alembic import op @@ -11,10 +11,10 @@ import flaschengeist # revision identifiers, used by Alembic. -revision = "f07df84f7a95" +revision = "98f2733bbe45" down_revision = None branch_labels = ("balance",) -depends_on = "d3026757c7cb" +depends_on = "flaschengeist" def upgrade(): diff --git a/flaschengeist/plugins/pricelist/migrations/7d9d306be676_initial_pricelist_migration.py b/flaschengeist/plugins/pricelist/migrations/58ab9b6a8839_pricelist_initial.py similarity index 97% rename from flaschengeist/plugins/pricelist/migrations/7d9d306be676_initial_pricelist_migration.py rename to flaschengeist/plugins/pricelist/migrations/58ab9b6a8839_pricelist_initial.py index c9da61e..3a6c5ad 100644 --- a/flaschengeist/plugins/pricelist/migrations/7d9d306be676_initial_pricelist_migration.py +++ b/flaschengeist/plugins/pricelist/migrations/58ab9b6a8839_pricelist_initial.py @@ -1,8 +1,8 @@ -"""Initial pricelist migration +"""pricelist: initial -Revision ID: 7d9d306be676 +Revision ID: 58ab9b6a8839 Revises: -Create Date: 2021-12-19 21:43:30.203811 +Create Date: 2022-02-23 14:45:30.563647 """ from alembic import op @@ -11,10 +11,10 @@ import flaschengeist # revision identifiers, used by Alembic. -revision = "7d9d306be676" +revision = "58ab9b6a8839" down_revision = None branch_labels = ("pricelist",) -depends_on = "d3026757c7cb" +depends_on = "flaschengeist" def upgrade(): diff --git a/flaschengeist/utils/plugin.py b/flaschengeist/utils/plugin.py deleted file mode 100644 index 56f0643..0000000 --- a/flaschengeist/utils/plugin.py +++ /dev/null @@ -1,49 +0,0 @@ -"""Plugin utils - -Utilities for handling Flaschengeist plugins -""" - -import pkg_resources -from flaschengeist import logger -from flaschengeist.plugins import Plugin - - -def get_plugins() -> list[type[Plugin]]: - """Get all installed plugins for Flaschengeist - - Returns: - list of classes implementing `flaschengeist.plugins.Plugin` - """ - logger.debug("Search for plugins") - - plugins = [] - for entry_point in pkg_resources.iter_entry_points("flaschengeist.plugins"): - try: - logger.debug(f"Found plugin: >{entry_point.name}<") - plugin_class = entry_point.load() - if issubclass(plugin_class, Plugin): - plugins.append(plugin_class) - except TypeError: - logger.error(f"Invalid entry point for plugin {entry_point.name} found.") - except pkg_resources.DistributionNotFound: - logger.warn(f"Requirements not fulfilled for {entry_point.name}") - logger.debug("DistributionNotFound", exc_info=True) - return plugins - - -def plugin_version(plugin: type[Plugin]) -> str: - """Get version of plugin - - Returns the version of a plugin, if plugin does not set the - version property, the version of the package providing the - plugin is taken. - - Args: - plugin: Plugin or Plugin class - - Returns: - Version as string - """ - if plugin.version: - return plugin.version - return pkg_resources.get_distribution(plugin.__module__.split(".", 1)[0]).version diff --git a/setup.cfg b/setup.cfg index 770ece4..41af0da 100644 --- a/setup.cfg +++ b/setup.cfg @@ -42,7 +42,7 @@ mysql = mysqlclient;platform_system!='Windows' [options.package_data] -* = *.toml +* = *.toml, script.py.mako [options.entry_points] console_scripts =