Add Alembic for migrations

MANIFEST.in

@@ -1,2 +1,4 @@
 include AUTHORS.txt COPYING.txt pylintrc ez_setup.py requirements.txt kittystore.spec .coveragerc
 graft kittystore/test/testdata
+include kittystore/sa/alembic.ini kittystore/sa/alembic/script.py.mako
+recursive-include kittystore/sa/alembic *.py

kittystore/__init__.py

@@ -76,12 +76,17 @@ def create_store(settings, debug=None):
         debug = getattr(settings, "KITTYSTORE_DEBUG", False)
 
     search_index = _get_search_index(settings)
-    from kittystore.storm import get_storm_store, create_storm_store
-    create_storm_store(settings, debug)
-    store = get_storm_store(settings, search_index, debug)
+    if getattr(settings, "KITTYSTORE_USE_STORM", False):
+        from kittystore.storm import get_storm_store, create_storm_db
+        version = create_storm_db(settings, debug)
+        store = get_storm_store(settings, search_index, debug)
+    else:
+        from kittystore.sa import create_sa_db, get_sa_store
+        version = create_sa_db(settings, debug)
+        store = get_sa_store(settings, search_index, debug)
     if search_index is not None and search_index.needs_upgrade():
         search_index.upgrade(store)
-    return store
+    return store, version
 
 
 class SchemaUpgradeNeeded(Exception):

kittystore/sa/__init__.py

@@ -1,20 +1,117 @@
 # -*- coding: utf-8 -*-
 
-from __future__ import absolute_import, with_statement
+from __future__ import absolute_import, with_statement, unicode_literals, print_function
 
+import os
+
+from pkg_resources import resource_filename
 from dogpile.cache import make_region
-from sqlalchemy import create_engine
+from sqlalchemy import create_engine, MetaData
 from sqlalchemy.orm import sessionmaker
 
+import alembic
+import alembic.config
+from alembic.script import ScriptDirectory
+from alembic.environment import EnvironmentContext
+
+from kittystore import SchemaUpgradeNeeded
 from kittystore.caching import setup_cache
 from .model import Base
 from .store import SAStore
 
 
+
+class SchemaManager(object):
+
+    def __init__(self, settings, engine=None, debug=False):
+        self.settings = settings
+        if engine is None:
+            engine = create_engine(settings.KITTYSTORE_URL, echo=debug)
+        self.engine = engine
+        self.debug = debug
+        self._config = None
+        self._script = None
+
+    @property
+    def config(self):
+        if self._config is None:
+            self._config = alembic.config.Config(resource_filename(
+                    "kittystore.sa", "alembic.ini"))
+            self._config.set_main_option(
+                    "sqlalchemy.url", self.settings.KITTYSTORE_URL)
+        return self._config
+
+    @property
+    def script(self):
+        if self._script is None:
+            self._script = ScriptDirectory.from_config(self.config)
+        return self._script
+
+    def check(self):
+        def _do_check(db_rev, context):
+            head_rev = self.script.get_current_head()
+            if db_rev == head_rev:
+                return [] # already at the head revision
+            raise SchemaUpgradeNeeded("version of the database: %s. Version of the code: %s" % (db_rev, head_rev))
+        with EnvironmentContext(self.config, self.script, fn=_do_check):
+            self.script.run_env()
+        return self.script.get_current_head()
+
+    def _db_is_storm(self):
+        md = MetaData()
+        md.reflect(bind=self.engine)
+        return "patch" in md.tables
+
+    def _create(self):
+        Base.metadata.create_all(self.engine)
+        alembic.command.stamp(self.config, "head")
+
+    def _upgrade(self):
+        alembic.command.upgrade(self.config, "head")
+
+    def setup_db(self):
+        # Alembic commands can't be run within the EnvironmentContext (they
+        # create their own), so we store the commands to run in this list and
+        # run them afterwards.
+        to_run = []
+        def _find_cmds(db_rev, context):
+            head_rev = self.script.get_current_head()
+            if db_rev == None:
+                if self._db_is_storm():
+                    # DB from a previous version, run migrations to remove
+                    # Storm-specific tables and upgrade to SQLAlchemy & Alembic
+                    to_run.append(self._upgrade)
+                else:
+                    # initial DB creation
+                    to_run.append(self._create)
+            elif db_rev != head_rev:
+                to_run.append(self._upgrade)
+            # db_rev == head_rev: already at the latest revision, nothing to do
+            return []
+        with EnvironmentContext(self.config, self.script, fn=_find_cmds):
+            self.script.run_env()
+        for cmd in to_run:
+            cmd()
+        return self.script.get_current_head()
+
+
+
+def create_sa_db(settings, debug=False):
+    engine = create_engine(settings.KITTYSTORE_URL, echo=debug)
+    schema_mgr = SchemaManager(settings, engine, debug)
+    return schema_mgr.setup_db()
+
+
+
 def get_sa_store(settings, search_index=None, debug=False, auto_create=False):
     engine = create_engine(settings.KITTYSTORE_URL, echo=debug)
-    if auto_create or True:
-        Base.metadata.create_all(engine)
+    schema_mgr = SchemaManager(settings, engine, debug)
+    try:
+        schema_mgr.check()
+    except SchemaUpgradeNeeded:
+        if not auto_create:
+            raise
+        schema_mgr.setup_db()
     Session = sessionmaker(bind=engine)
     session = Session()
     cache = make_region()

kittystore/sa/alembic.ini

@@ -0,0 +1,59 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = %(here)s/alembic
+
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# max length of characters to apply to the
+# "slug" field
+#truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+sqlalchemy.url = driver://user:pass@localhost/dbname
+
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = WARN
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

kittystore/sa/alembic/env.py

@@ -0,0 +1,73 @@
+from __future__ import with_statement
+from alembic import context
+from sqlalchemy import engine_from_config, pool
+from logging.config import fileConfig
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+fileConfig(config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
+#target_metadata = None
+import kittystore
+target_metadata = kittystore.sa.model.Base.metadata
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+def run_migrations_offline():
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(url=url, target_metadata=target_metadata)
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+def run_migrations_online():
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+    engine = engine_from_config(
+                config.get_section(config.config_ini_section),
+                prefix='sqlalchemy.',
+                poolclass=pool.NullPool)
+
+    connection = engine.connect()
+    context.configure(
+                connection=connection,
+                target_metadata=target_metadata
+                )
+
+    try:
+        with context.begin_transaction():
+            context.run_migrations()
+    finally:
+        connection.close()
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()
+

kittystore/sa/alembic/script.py.mako

@@ -0,0 +1,22 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision}
+Create Date: ${create_date}
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+def upgrade():
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade():
+    ${downgrades if downgrades else "pass"}

kittystore/sa/alembic/versions/31959b2e8f44_initial.py

@@ -0,0 +1,27 @@
+"""Initial migration (empty)
+
+This empty migration file makes sure there is always an alembic_version in the
+database. As a consequence, if the DB version is reported as None, it means the
+database needs to be created from scratch with SQLAlchemy itself.
+
+It also removes the "patch" table left over from Storm (if it exists).
+
+Revision ID: 31959b2e8f44
+Revises: None
+Create Date: 2014-08-13 15:07:09.282855
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '31959b2e8f44'
+down_revision = None
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    op.drop_table('patch') # Storm migration versionning table
+
+def downgrade():
+    pass

kittystore/scripts.py

@@ -97,11 +97,7 @@ def updatedb():
         store = get_store(settings, debug=opts.debug)
     except SchemaUpgradeNeeded:
         print "Upgrading the schema..."
-        store = create_store(settings, debug=opts.debug)
-        version = list(store.db.execute(
-                    "SELECT patch.version FROM patch "
-                    "ORDER BY version DESC LIMIT 1"
-                    ))[0][0]
+        store, version = create_store(settings, debug=opts.debug)
         print "Done, the current schema version is %d." % version
         print "Synchonizing data from Mailman, this can take some time..."
         sync_mailman(store)

kittystore/storm/__init__.py

@@ -24,12 +24,17 @@ def _get_schema(settings):
     return CheckingSchema(schema.CREATES[dbtype], [], [], schema)
 
 
-def create_storm_store(settings, debug=False):
+def create_storm_db(settings, debug=False):
     if debug:
         storm.tracer.debug(True, stream=sys.stdout)
     store = _get_native_store(settings)
     dbschema = _get_schema(settings)
     dbschema.upgrade(store)
+    version = list(store.execute(
+                "SELECT patch.version FROM patch "
+                "ORDER BY version DESC LIMIT 1"
+                ))[0][0]
+    return version
 
 
 def get_storm_store(settings, search_index=None, debug=False, auto_create=False):