Merge pull request #871 from MolSSI/spec_hash

Improve the way specifications are added to the database

qcarchivetesting/qcarchivetesting/helpers.py

@@ -164,12 +164,12 @@ def load_ip_test_data():
     return ret
 
 
-def load_hash_test_data():
+def load_hash_test_data(file_base: str):
     """
     Loads data for testing dictionary hashing
     """
 
-    file_path = os.path.join(_my_path, "hash_data", "dict_hash_test_data.json.xz")
+    file_path = os.path.join(_my_path, "hash_data", f"{file_base}.json.xz")
     with lzma.open(file_path, "rt") as f:
         return json.load(f)
 

qcfractal/qcfractal/alembic/versions/2024-12-17-12d37bad13ba_gridoptimization_spec_hashing.py

@@ -0,0 +1,60 @@
+"""Gridoptimization spec hashing
+
+Revision ID: 12d37bad13ba
+Revises: 518ee94f4749
+Create Date: 2024-12-17 15:41:38.740543
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+from migration_helpers.hashing import hash_dict_1
+
+# revision identifiers, used by Alembic.
+revision = "12d37bad13ba"
+down_revision = "518ee94f4749"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column(
+        "gridoptimization_specification", sa.Column("protocols", postgresql.JSONB(astext_type=sa.Text()), nullable=True)
+    )
+    op.add_column("gridoptimization_specification", sa.Column("specification_hash", sa.String(), nullable=True))
+
+    # Generate hashes for existing specs
+    conn = op.get_bind()
+    res = conn.execute(sa.text(f"SELECT id, program, keywords FROM gridoptimization_specification;"))
+    all_spec = res.fetchall()
+    for spec_id, program, keywords in all_spec:
+        d = {
+            "program": program,
+            "keywords": keywords,
+            "protocols": {},  # Default for now
+        }
+        h = hash_dict_1(d)
+        op.execute(
+            sa.text(f"""UPDATE gridoptimization_specification SET specification_hash = '{h}' WHERE id = {spec_id};""")
+        )
+
+    op.execute(sa.text("UPDATE gridoptimization_specification SET protocols = '{}'::JSONB"))
+
+    op.alter_column("gridoptimization_specification", "specification_hash", nullable=False)
+    op.alter_column("gridoptimization_specification", "protocols", nullable=False)
+
+    op.drop_constraint("ux_gridoptimization_specification_keys", "gridoptimization_specification", type_="unique")
+    op.drop_column("gridoptimization_specification", "keywords_hash")
+    op.create_unique_constraint(
+        "ux_gridoptimization_specification_keys",
+        "gridoptimization_specification",
+        ["specification_hash", "optimization_specification_id"],
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    raise RuntimeError("Downgrade not supported.")

qcfractal/qcfractal/alembic/versions/2024-12-17-518ee94f4749_torsiondrive_spec_hashing.py

@@ -0,0 +1,59 @@
+"""Torsiondrive spec hashing
+
+Revision ID: 518ee94f4749
+Revises: e04bf1eedfbc
+Create Date: 2024-12-17 15:18:40.252669
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+from migration_helpers.hashing import hash_dict_1
+
+# revision identifiers, used by Alembic.
+revision = "518ee94f4749"
+down_revision = "e04bf1eedfbc"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column(
+        "torsiondrive_specification", sa.Column("protocols", postgresql.JSONB(astext_type=sa.Text()), nullable=True)
+    )
+    op.add_column("torsiondrive_specification", sa.Column("specification_hash", sa.String(), nullable=True))
+
+    # Generate hashes for existing specs
+    conn = op.get_bind()
+    res = conn.execute(sa.text(f"SELECT id, program, keywords FROM torsiondrive_specification;"))
+    all_spec = res.fetchall()
+    for spec_id, program, keywords in all_spec:
+        d = {
+            "program": program,
+            "keywords": keywords,
+            "protocols": {},  # Default for now
+        }
+        h = hash_dict_1(d)
+        op.execute(
+            sa.text(f"""UPDATE torsiondrive_specification SET specification_hash = '{h}' WHERE id = {spec_id};""")
+        )
+
+    op.execute(sa.text("UPDATE torsiondrive_specification SET protocols = '{}'::JSONB"))
+
+    op.alter_column("torsiondrive_specification", "specification_hash", nullable=False)
+    op.alter_column("torsiondrive_specification", "protocols", nullable=False)
+    op.drop_constraint("ux_torsiondrive_specification_keys", "torsiondrive_specification", type_="unique")
+    op.drop_column("torsiondrive_specification", "keywords_hash")
+    op.create_unique_constraint(
+        "ux_torsiondrive_specification_keys",
+        "torsiondrive_specification",
+        ["specification_hash", "optimization_specification_id"],
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    raise RuntimeError("Downgrade not supported.")

qcfractal/qcfractal/alembic/versions/2024-12-17-6b758fd53ff0_singlepoint_spec_hashing.py

@@ -0,0 +1,52 @@
+"""Singlepoint spec hashing
+
+Revision ID: 6b758fd53ff0
+Revises: 8263992eb6c8
+Create Date: 2024-12-17 10:05:11.756731
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+from migration_helpers.hashing import hash_dict_1
+
+# revision identifiers, used by Alembic.
+revision = "6b758fd53ff0"
+down_revision = "8263992eb6c8"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column("qc_specification", sa.Column("specification_hash", sa.String(), nullable=True))
+
+    # Generate hashes for existing specs
+    conn = op.get_bind()
+    res = conn.execute(
+        sa.text(f"SELECT id, program, driver, method, basis, keywords, protocols FROM qc_specification;")
+    )
+    all_spec = res.fetchall()
+
+    for spec_id, program, driver, method, basis, keywords, protocols in all_spec:
+        d = {
+            "program": program,
+            "driver": driver,
+            "method": method,
+            "basis": basis,
+            "keywords": keywords,
+            "protocols": protocols,
+        }
+        h = hash_dict_1(d)
+        op.execute(sa.text(f"""UPDATE qc_specification SET specification_hash = '{h}' WHERE id = {spec_id};"""))
+
+    op.alter_column("qc_specification", "specification_hash", nullable=False)
+    op.drop_constraint("ux_qc_specification_keys", "qc_specification", type_="unique")
+    op.drop_column("qc_specification", "keywords_hash")
+    op.create_unique_constraint("ux_qc_specification_specification_hash", "qc_specification", ["specification_hash"])
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    raise RuntimeError("Downgrade not supported.")

qcfractal/qcfractal/alembic/versions/2024-12-17-a3c51b03bc19_reaction_spec_hashing.py

@@ -0,0 +1,56 @@
+"""Reaction spec hashing
+
+Revision ID: a3c51b03bc19
+Revises: 12d37bad13ba
+Create Date: 2024-12-17 17:12:43.414568
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+from migration_helpers.hashing import hash_dict_1
+
+# revision identifiers, used by Alembic.
+revision = "a3c51b03bc19"
+down_revision = "12d37bad13ba"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column(
+        "reaction_specification", sa.Column("protocols", postgresql.JSONB(astext_type=sa.Text()), nullable=True)
+    )
+    op.add_column("reaction_specification", sa.Column("specification_hash", sa.String(), nullable=True))
+
+    # Generate hashes for existing specs
+    conn = op.get_bind()
+    res = conn.execute(sa.text(f"SELECT id, program, keywords FROM reaction_specification;"))
+    all_spec = res.fetchall()
+    for spec_id, program, keywords in all_spec:
+        d = {
+            "program": program,
+            "keywords": keywords,
+            "protocols": {},  # Default for now
+        }
+        h = hash_dict_1(d)
+        op.execute(sa.text(f"""UPDATE reaction_specification SET specification_hash = '{h}' WHERE id = {spec_id};"""))
+    op.execute(sa.text("UPDATE reaction_specification SET protocols = '{}'::JSONB"))
+
+    op.alter_column("reaction_specification", "protocols", nullable=False)
+    op.alter_column("reaction_specification", "specification_hash", nullable=False)
+    op.drop_constraint("ux_reaction_specification_keys", "reaction_specification", type_="unique")
+    op.drop_column("reaction_specification", "keywords_hash")
+    op.create_unique_constraint(
+        "ux_reaction_specification_keys",
+        "reaction_specification",
+        ["specification_hash", "singlepoint_specification_id", "optimization_specification_id"],
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    raise RuntimeError("Downgrade not supported.")

qcfractal/qcfractal/alembic/versions/2024-12-17-e04bf1eedfbc_optimization_spec_hashing.py

@@ -0,0 +1,52 @@
+"""Optimization spec hashing
+
+Revision ID: e04bf1eedfbc
+Revises: 6b758fd53ff0
+Create Date: 2024-12-17 10:50:37.251828
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+from migration_helpers.hashing import hash_dict_1
+
+# revision identifiers, used by Alembic.
+revision = "e04bf1eedfbc"
+down_revision = "6b758fd53ff0"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column("optimization_specification", sa.Column("specification_hash", sa.String(), nullable=True))
+
+    # Generate hashes for existing specs
+    conn = op.get_bind()
+    res = conn.execute(sa.text(f"SELECT id, program, keywords, protocols FROM optimization_specification;"))
+    all_spec = res.fetchall()
+    for spec_id, program, keywords, protocols in all_spec:
+        d = {
+            "program": program,
+            "keywords": keywords,
+            "protocols": protocols,
+        }
+        h = hash_dict_1(d)
+        op.execute(
+            sa.text(f"""UPDATE optimization_specification SET specification_hash = '{h}' WHERE id = {spec_id};""")
+        )
+
+    op.alter_column("optimization_specification", "specification_hash", nullable=False)
+    op.drop_constraint("ux_optimization_specification_keys", "optimization_specification", type_="unique")
+    op.drop_column("optimization_specification", "keywords_hash")
+    op.create_unique_constraint(
+        "ux_optimization_specification_keys",
+        "optimization_specification",
+        ["specification_hash", "qc_specification_id"],
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    raise RuntimeError("Downgrade not supported.")

qcfractal/qcfractal/alembic/versions/2024-12-18-0587bb0220aa_neb_spec_hashing.py

@@ -0,0 +1,60 @@
+"""NEB spec hashing
+
+Revision ID: 0587bb0220aa
+Revises: a3c51b03bc19
+Create Date: 2024-12-18 10:18:17.162740
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+from migration_helpers.hashing import hash_dict_1
+
+# revision identifiers, used by Alembic.
+revision = "0587bb0220aa"
+down_revision = "a3c51b03bc19"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column("neb_specification", sa.Column("specification_hash", sa.String(), nullable=True))
+    op.add_column("neb_specification", sa.Column("protocols", postgresql.JSONB(astext_type=sa.Text()), nullable=True))
+
+    # Generate hashes for existing specs
+    conn = op.get_bind()
+    res = conn.execute(sa.text(f"SELECT id, program, keywords FROM neb_specification;"))
+    all_spec = res.fetchall()
+    for spec_id, program, keywords in all_spec:
+        d = {
+            "program": program,
+            "keywords": keywords,
+            "protocols": {},  # Default for now
+        }
+        h = hash_dict_1(d)
+        op.execute(sa.text(f"""UPDATE neb_specification SET specification_hash = '{h}' WHERE id = {spec_id};"""))
+
+    op.execute(sa.text("UPDATE neb_specification SET protocols = '{}'::JSONB"))
+
+    op.alter_column("neb_specification", "protocols", nullable=False)
+    op.alter_column("neb_specification", "specification_hash", nullable=False)
+    op.drop_constraint("ux_neb_specification_keys", "neb_specification", type_="unique")
+    op.create_unique_constraint(
+        "ux_neb_specification_keys",
+        "neb_specification",
+        ["specification_hash", "singlepoint_specification_id", "optimization_specification_id"],
+    )
+    op.drop_column("neb_specification", "keywords_hash")
+
+    op.create_check_constraint(
+        "ck_neb_specification_program_lower", "neb_specification", sa.text("program = LOWER(program)")
+    )
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    raise RuntimeError("Downgrade not supported.")

qcfractal/qcfractal/components/gridoptimization/record_db_models.py

@@ -57,21 +57,19 @@ class GridoptimizationSpecificationORM(BaseORM):
     __tablename__ = "gridoptimization_specification"
 
     id = Column(Integer, primary_key=True)
+    specification_hash = Column(String, nullable=False)
 
     program = Column(String(100), nullable=False)
 
     optimization_specification_id = Column(Integer, ForeignKey(OptimizationSpecificationORM.id), nullable=False)
     optimization_specification = relationship(OptimizationSpecificationORM, lazy="joined")
 
     keywords = Column(JSONB, nullable=False)
-    keywords_hash = Column(String, nullable=False)
+    protocols = Column(JSONB, nullable=False)
 
     __table_args__ = (
         UniqueConstraint(
-            "program",
-            "optimization_specification_id",
-            "keywords_hash",
-            name="ux_gridoptimization_specification_keys",
+            "specification_hash", "optimization_specification_id", name="ux_gridoptimization_specification_keys"
         ),
         Index("ix_gridoptimization_specification_program", "program"),
         Index("ix_gridoptimization_specification_optimization_specification_id", "optimization_specification_id"),
@@ -81,7 +79,13 @@ class GridoptimizationSpecificationORM(BaseORM):
         CheckConstraint("program = LOWER(program)", name="ck_gridoptimization_specification_program_lower"),
     )
 
-    _qcportal_model_excludes = ["id", "keywords_hash", "optimization_specification_id"]
+    # TODO - protocols will be in model eventually
+    _qcportal_model_excludes = [
+        "id",
+        "specification_hash",
+        "optimization_specification_id",
+        "protocols",
+    ]
 
     @property
     def short_description(self) -> str: