Parallelize database handling (breaking, fixes #49) (#119)

- now uses `Future` just to parallelize all calls to postgres - cleaner separation of input and processing - now needs a `DataSource` instead of a `Connection`, which breaks API - adds a dependency upon hikari - keep it at hikari version 4 to avoid alpha slf4j - add slf4j-nop to avoid spam
oyvindberg · Jul 13, 2024 · b6db739 · b6db739
1 parent 2b83b5c
commit b6db739
Show file tree

Hide file tree

Showing 13 changed files with 240 additions and 139 deletions.
diff --git a/bleep.yaml b/bleep.yaml
@@ -15,6 +15,8 @@ projects:
     - com.typesafe.play::play-json:2.10.6
     - org.playframework.anorm::anorm:2.7.0
     - org.postgresql:postgresql:42.7.3
+    - com.zaxxer:HikariCP:4.0.3
+    - org.slf4j:slf4j-nop:2.0.13
     extends: template-cross
     platform:
       mainClass: com.foo.App

diff --git a/site-in/other-features/generate-into-multiple-projects.md b/site-in/other-features/generate-into-multiple-projects.md
@@ -19,10 +19,11 @@ import typo.*
 import java.nio.file.Path
 import java.sql.Connection
 
-def generate(implicit c: Connection): String = {
+def generate(ds: TypoDataSource): String = {
   val cwd: Path = Path.of(sys.props("user.dir"))
 
   val generated = generateFromDb(
+    ds,
     Options(
       pkg = "org.mypkg",
       jsonLibs = Nil,

diff --git a/site-in/setup.md b/site-in/setup.md
@@ -40,8 +40,13 @@ put it in `gen-db.sc` and run `scala-cli gen-db.sc`
 import typo.*
 
 // adapt to your instance and credentials
-implicit val c: java.sql.Connection =
-  java.sql.DriverManager.getConnection("jdbc:postgresql://localhost:6432/postgres?user=postgres&password=password")
+val ds = TypoDataSource.hikari(
+  server = "localhost", 
+  port = 6432, 
+  databaseName = "Adventureworks", 
+  username = "postgres", 
+  password = "password"
+)
 
 val options = Options(
   // customize package name for generated code
@@ -67,6 +72,7 @@ val scriptsFolder = location.resolve("sql")
 val selector = Selector.ExcludePostgresInternal
 
 generateFromDb(
+  ds,
   options, 
   targetFolder = targetDir,
   testTargetFolder = Some(testTargetDir),

diff --git a/typo-scripts/src/scala/scripts/CompileBenchmark.scala b/typo-scripts/src/scala/scripts/CompileBenchmark.scala
@@ -8,8 +8,10 @@ import typo.internal.generate
 import typo.internal.sqlfiles.readSqlFileDirectories
 
 import java.nio.file.Path
-import java.sql.{Connection, DriverManager}
 import scala.annotation.nowarn
+import scala.concurrent.Await
+import scala.concurrent.ExecutionContext.Implicits.global
+import scala.concurrent.duration.Duration
 
 object CompileBenchmark extends BleepScript("CompileBenchmark") {
   val buildDir = Path.of(sys.props("user.dir"))
@@ -25,10 +27,9 @@ object CompileBenchmark extends BleepScript("CompileBenchmark") {
   )
 
   override def run(started: Started, commands: Commands, args: List[String]): Unit = {
-    implicit val c: Connection = DriverManager.getConnection(
-      "jdbc:postgresql://localhost:6432/Adventureworks?user=postgres&password=password"
-    )
-    val metadb = MetaDb.fromDb(TypoLogger.Noop)
+    val ds = TypoDataSource.hikari(server = "localhost", port = 6432, databaseName = "Adventureworks", username = "postgres", password = "password")
+    val metadb = Await.result(MetaDb.fromDb(logger = TypoLogger.Noop, ds = ds, viewSelector = Selector.All), Duration.Inf)
+    val sqlFiles = Await.result(readSqlFileDirectories(TypoLogger.Noop, buildDir.resolve("adventureworks_sql"), ds), Duration.Inf)
 
     val crossIds = List(
       "jvm212",
@@ -69,7 +70,7 @@ object CompileBenchmark extends BleepScript("CompileBenchmark") {
               targetSources,
               None,
               Selector.ExcludePostgresInternal, // All
-              readSqlFileDirectories(TypoLogger.Noop, buildDir.resolve("adventureworks_sql")),
+              sqlFiles,
               Nil
             )
           ).foreach(_.overwriteFolder())

diff --git a/typo-scripts/src/scala/scripts/GeneratedAdventureWorks.scala b/typo-scripts/src/scala/scripts/GeneratedAdventureWorks.scala
@@ -7,8 +7,10 @@ import typo.internal.sqlfiles.readSqlFileDirectories
 import typo.internal.{FileSync, generate}
 
 import java.nio.file.Path
-import java.sql.{Connection, DriverManager}
 import java.util.concurrent.atomic.AtomicReference
+import scala.concurrent.Await
+import scala.concurrent.ExecutionContext.Implicits.global
+import scala.concurrent.duration.Duration
 
 object GeneratedAdventureWorks {
   val buildDir = Path.of(sys.props("user.dir"))
@@ -22,11 +24,10 @@ object GeneratedAdventureWorks {
       .map(_.minLogLevel(LogLevel.info))
       .untyped
       .use { logger =>
-        implicit val c: Connection = DriverManager.getConnection(
-          "jdbc:postgresql://localhost:6432/Adventureworks?user=postgres&password=password"
-        )
+        val ds = TypoDataSource.hikari(server = "localhost", port = 6432, databaseName = "Adventureworks", username = "postgres", password = "password")
         val scriptsPath = buildDir.resolve("adventureworks_sql")
-        val metadb = MetaDb.fromDb(TypoLogger.Console)
+        val selector = Selector.ExcludePostgresInternal
+        val metadb = Await.result(MetaDb.fromDb(TypoLogger.Console, ds, selector), Duration.Inf)
 
         val variants = List(
           (DbLibName.Anorm, JsonLibName.PlayJson, "typo-tester-anorm", new AtomicReference(Map.empty[RelPath, sc.Code])),
@@ -35,7 +36,7 @@ object GeneratedAdventureWorks {
         )
 
         def go(): Unit = {
-          val newSqlScripts = readSqlFileDirectories(TypoLogger.Console, scriptsPath)
+          val newSqlScripts = Await.result(readSqlFileDirectories(TypoLogger.Console, scriptsPath, ds), Duration.Inf)
 
           variants.foreach { case (dbLib, jsonLib, projectPath, oldFilesRef) =>
             val options = Options(
@@ -55,7 +56,7 @@ object GeneratedAdventureWorks {
             val targetSources = buildDir.resolve(s"$projectPath/generated-and-checked-in")
 
             val newFiles: Generated =
-              generate(options, metadb, ProjectGraph(name = "", targetSources, None, Selector.ExcludePostgresInternal, newSqlScripts, Nil)).head
+              generate(options, metadb, ProjectGraph(name = "", targetSources, None, selector, newSqlScripts, Nil)).head
 
             val knownUnchanged: Set[RelPath] = {
               val oldFiles = oldFilesRef.get()

diff --git a/typo-scripts/src/scala/scripts/GeneratedSources.scala b/typo-scripts/src/scala/scripts/GeneratedSources.scala
@@ -3,20 +3,11 @@ package scripts
 import typo.*
 
 import java.nio.file.Path
-import java.sql.{Connection, DriverManager}
-import java.util
 import scala.annotation.nowarn
 
 object GeneratedSources {
   def main(args: Array[String]): Unit = {
-    implicit val c: Connection = {
-      val url = "jdbc:postgresql://localhost:6432/postgres"
-      val props = new util.Properties
-      props.setProperty("user", "postgres")
-      props.setProperty("password", "password")
-      props.setProperty("port", "6432")
-      DriverManager.getConnection(url, props)
-    }
+    val ds = TypoDataSource.hikari(server = "localhost", port = 6432, databaseName = "Adventureworks", username = "postgres", password = "password")
 
     val header =
       """|/**
@@ -32,6 +23,7 @@ object GeneratedSources {
     val typoSources = buildDir.resolve("typo/generated-and-checked-in")
 
     val files = generateFromDb(
+      ds,
       Options(
         pkg = "typo.generated",
         jsonLibs = List(JsonLibName.PlayJson),

diff --git a/typo/src/scala/typo/MetaDb.scala b/typo/src/scala/typo/MetaDb.scala
@@ -4,27 +4,26 @@ import typo.generated.custom.comments.{CommentsSqlRepoImpl, CommentsSqlRow}
 import typo.generated.custom.constraints.{ConstraintsSqlRepoImpl, ConstraintsSqlRow}
 import typo.generated.custom.domains.{DomainsSqlRepoImpl, DomainsSqlRow}
 import typo.generated.custom.enums.{EnumsSqlRepoImpl, EnumsSqlRow}
-import typo.generated.custom.view_find_all.*
 import typo.generated.custom.table_comments.*
+import typo.generated.custom.view_find_all.*
 import typo.generated.information_schema.columns.{ColumnsViewRepoImpl, ColumnsViewRow}
 import typo.generated.information_schema.key_column_usage.{KeyColumnUsageViewRepoImpl, KeyColumnUsageViewRow}
 import typo.generated.information_schema.referential_constraints.{ReferentialConstraintsViewRepoImpl, ReferentialConstraintsViewRow}
 import typo.generated.information_schema.table_constraints.{TableConstraintsViewRepoImpl, TableConstraintsViewRow}
 import typo.generated.information_schema.tables.{TablesViewRepoImpl, TablesViewRow}
-import typo.internal.analysis.{DecomposedSql, JdbcMetadata, MaybeReturnsRows, NullabilityFromExplain, ParsedName}
+import typo.internal.analysis.*
 import typo.internal.metadb.{Enums, ForeignKeys, PrimaryKeys, UniqueKeys}
 import typo.internal.{DebugJson, Lazy, TypeMapperDb}
 
-import java.sql.Connection
 import scala.collection.immutable.SortedSet
+import scala.concurrent.{ExecutionContext, Future}
 
 case class MetaDb(
     relations: Map[db.RelationName, Lazy[db.Relation]],
     enums: List[db.StringEnum],
     domains: List[db.Domain]
 ) {
   val typeMapperDb = TypeMapperDb(enums, domains)
-
 }
 
 object MetaDb {
@@ -35,42 +34,84 @@ object MetaDb {
       pgEnums: List[EnumsSqlRow],
       tables: List[TablesViewRow],
       columns: List[ColumnsViewRow],
-      views: List[ViewFindAllSqlRow],
+      views: Map[db.RelationName, AnalyzedView],
       domains: List[DomainsSqlRow],
       columnComments: List[CommentsSqlRow],
       constraints: List[ConstraintsSqlRow],
       tableComments: List[TableCommentsSqlRow]
   )
 
+  case class AnalyzedView(
+      row: ViewFindAllSqlRow,
+      decomposedSql: DecomposedSql,
+      jdbcMetadata: JdbcMetadata,
+      nullabilityAnalysis: NullabilityFromExplain.NullableColumns
+  )
+
   object Input {
-    def fromDb(logger: TypoLogger)(implicit c: Connection): Input = {
-      def timed[T](name: String)(f: => T): T = {
-        val start = System.currentTimeMillis()
-        val result = f
-        val end = System.currentTimeMillis()
-        logger.info(s"fetched $name from PG in ${end - start}ms")
-        result
+    def fromDb(logger: TypoLogger, ds: TypoDataSource, viewSelector: Selector)(implicit ev: ExecutionContext): Future[Input] = {
+      val tableConstraints = logger.timed("fetching tableConstraints")(ds.run(implicit c => (new TableConstraintsViewRepoImpl).selectAll))
+      val keyColumnUsage = logger.timed("fetching keyColumnUsage")(ds.run(implicit c => (new KeyColumnUsageViewRepoImpl).selectAll))
+      val referentialConstraints = logger.timed("fetching referentialConstraints")(ds.run(implicit c => (new ReferentialConstraintsViewRepoImpl).selectAll))
+      val pgEnums = logger.timed("fetching pgEnums")(ds.run(implicit c => (new EnumsSqlRepoImpl)()))
+      val tables = logger.timed("fetching tables")(ds.run(implicit c => (new TablesViewRepoImpl).selectAll.filter(_.tableType.contains("BASE TABLE"))))
+      val columns = logger.timed("fetching columns")(ds.run(implicit c => (new ColumnsViewRepoImpl).selectAll))
+      val views = logger.timed("fetching and analyzing views")(ds.run(implicit c => (new ViewFindAllSqlRepoImpl)())).flatMap { viewRows =>
+        val analyzedRows: List[Future[(db.RelationName, AnalyzedView)]] = viewRows.flatMap { viewRow =>
+          val name = db.RelationName(viewRow.tableSchema, viewRow.tableName.get)
+          if (viewRow.viewDefinition.isDefined && viewSelector.include(name)) Some {
+            val sqlContent = viewRow.viewDefinition.get
+            val decomposedSql = DecomposedSql.parse(sqlContent)
+            val jdbcMetadata = ds.run(implicit c => JdbcMetadata.from(sqlContent))
+            val nullabilityAnalysis = ds.run(implicit c => NullabilityFromExplain.from(decomposedSql, Nil))
+            for {
+              jdbcMetadata <- jdbcMetadata.map {
+                case Left(str)    => sys.error(str)
+                case Right(value) => value
+              }
+              nullabilityAnalysis <- nullabilityAnalysis
+            } yield name -> AnalyzedView(viewRow, decomposedSql, jdbcMetadata, nullabilityAnalysis)
+          }
+          else None
+        }
+        Future.sequence(analyzedRows).map(_.toMap)
       }
-
-      Input(
-        tableConstraints = timed("tableConstraints")((new TableConstraintsViewRepoImpl).selectAll),
-        keyColumnUsage = timed("keyColumnUsage")((new KeyColumnUsageViewRepoImpl).selectAll),
-        referentialConstraints = timed("referentialConstraints")((new ReferentialConstraintsViewRepoImpl).selectAll),
-        pgEnums = timed("pgEnums")((new EnumsSqlRepoImpl)()),
-        tables = timed("tables")((new TablesViewRepoImpl).selectAll.filter(_.tableType.contains("BASE TABLE"))),
-        columns = timed("columns")((new ColumnsViewRepoImpl).selectAll),
-        views = timed("views")((new ViewFindAllSqlRepoImpl)()),
-        domains = timed("domains")((new DomainsSqlRepoImpl)()),
-        columnComments = timed("columnComments")((new CommentsSqlRepoImpl)()),
-        constraints = timed("constraints")((new ConstraintsSqlRepoImpl)()),
-        tableComments = timed("tableComments")((new TableCommentsSqlRepoImpl)())
+      val domains = logger.timed("fetching domains")(ds.run(implicit c => (new DomainsSqlRepoImpl)()))
+      val columnComments = logger.timed("fetching columnComments")(ds.run(implicit c => (new CommentsSqlRepoImpl)()))
+      val constraints = logger.timed("fetching constraints")(ds.run(implicit c => (new ConstraintsSqlRepoImpl)()))
+      val tableComments = logger.timed("fetching tableComments")(ds.run(implicit c => (new TableCommentsSqlRepoImpl)()))
+      for {
+        tableConstraints <- tableConstraints
+        keyColumnUsage <- keyColumnUsage
+        referentialConstraints <- referentialConstraints
+        pgEnums <- pgEnums
+        tables <- tables
+        columns <- columns
+        views <- views
+        domains <- domains
+        columnComments <- columnComments
+        constraints <- constraints
+        tableComments <- tableComments
+      } yield Input(
+        tableConstraints,
+        keyColumnUsage,
+        referentialConstraints,
+        pgEnums,
+        tables,
+        columns,
+        views,
+        domains,
+        columnComments,
+        constraints,
+        tableComments
       )
     }
   }
 
-  def fromDb(logger: TypoLogger)(implicit c: Connection): MetaDb = {
-    val input = Input.fromDb(logger)
+  def fromDb(logger: TypoLogger, ds: TypoDataSource, viewSelector: Selector)(implicit ec: ExecutionContext): Future[MetaDb] =
+    Input.fromDb(logger, ds, viewSelector).map(input => fromInput(logger, input))
 
+  def fromInput(logger: TypoLogger, input: Input): MetaDb = {
     val foreignKeys = ForeignKeys(input.tableConstraints, input.keyColumnUsage, input.referentialConstraints)
     val primaryKeys = PrimaryKeys(input.tableConstraints, input.keyColumnUsage)
     val uniqueKeys = UniqueKeys(input.tableConstraints, input.keyColumnUsage)
@@ -114,59 +155,52 @@ object MetaDb {
     val tableCommentsByTable: Map[db.RelationName, String] =
       input.tableComments.flatMap(c => c.description.map(d => (db.RelationName(Some(c.schema), c.name), d))).toMap
     val views: Map[db.RelationName, Lazy[db.View]] =
-      input.views.flatMap { viewRow =>
-        viewRow.viewDefinition.map { sqlContent =>
-          val relationName = db.RelationName(viewRow.tableSchema, viewRow.tableName.get)
-          val lazyAnalysis = Lazy {
-            logger.info(s"Analyzing view ${relationName.value}")
-            val decomposedSql = DecomposedSql.parse(sqlContent)
-            val Right(jdbcMetadata) = JdbcMetadata.from(sqlContent): @unchecked
-            val nullabilityInfo = NullabilityFromExplain.from(decomposedSql, Nil).nullableIndices
-            val deps: Map[db.ColName, List[(db.RelationName, db.ColName)]] =
-              jdbcMetadata.columns match {
-                case MaybeReturnsRows.Query(columns) =>
-                  columns.toList.flatMap(col => col.baseRelationName.zip(col.baseColumnName).map(t => col.name -> List(t))).toMap
-                case MaybeReturnsRows.Update =>
-                  Map.empty
-              }
+      input.views.map { case (relationName, AnalyzedView(viewRow, decomposedSql, jdbcMetadata, nullabilityAnalysis)) =>
+        val lazyAnalysis = Lazy {
+          val deps: Map[db.ColName, List[(db.RelationName, db.ColName)]] =
+            jdbcMetadata.columns match {
+              case MaybeReturnsRows.Query(columns) =>
+                columns.toList.flatMap(col => col.baseRelationName.zip(col.baseColumnName).map(t => col.name -> List(t))).toMap
+              case MaybeReturnsRows.Update =>
+                Map.empty
+            }
 
-            val cols: NonEmptyList[(db.Col, ParsedName)] =
-              jdbcMetadata.columns match {
-                case MaybeReturnsRows.Query(metadataCols) =>
-                  metadataCols.zipWithIndex.map { case (mdCol, idx) =>
-                    val nullability: Nullability =
-                      mdCol.parsedColumnName.nullability.getOrElse {
-                        if (nullabilityInfo.exists(_.values(idx))) Nullability.Nullable
-                        else mdCol.isNullable.toNullability
-                      }
-
-                    val dbType = typeMapperDb.dbTypeFrom(mdCol.columnTypeName, Some(mdCol.precision)) { () =>
-                      logger.warn(s"Couldn't translate type from view ${relationName.value} column ${mdCol.name.value} with type ${mdCol.columnTypeName}. Falling back to text")
+          val cols: NonEmptyList[(db.Col, ParsedName)] =
+            jdbcMetadata.columns match {
+              case MaybeReturnsRows.Query(metadataCols) =>
+                metadataCols.zipWithIndex.map { case (mdCol, idx) =>
+                  val nullability: Nullability =
+                    mdCol.parsedColumnName.nullability.getOrElse {
+                      if (nullabilityAnalysis.nullableIndices.exists(_.values(idx))) Nullability.Nullable
+                      else mdCol.isNullable.toNullability
                     }
 
-                    val coord = (relationName, mdCol.name)
-                    val dbCol = db.Col(
-                      parsedName = mdCol.parsedColumnName,
-                      tpe = dbType,
-                      udtName = None,
-                      columnDefault = None,
-                      identity = None,
-                      comment = comments.get(coord),
-                      jsonDescription = DebugJson(mdCol),
-                      nullability = nullability,
-                      constraints = constraints.getOrElse(coord, Nil)
-                    )
-                    (dbCol, mdCol.parsedColumnName)
+                  val dbType = typeMapperDb.dbTypeFrom(mdCol.columnTypeName, Some(mdCol.precision)) { () =>
+                    logger.warn(s"Couldn't translate type from view ${relationName.value} column ${mdCol.name.value} with type ${mdCol.columnTypeName}. Falling back to text")
                   }
 
-                case MaybeReturnsRows.Update => ???
-              }
+                  val coord = (relationName, mdCol.name)
+                  val dbCol = db.Col(
+                    parsedName = mdCol.parsedColumnName,
+                    tpe = dbType,
+                    udtName = None,
+                    columnDefault = None,
+                    identity = None,
+                    comment = comments.get(coord),
+                    jsonDescription = DebugJson(mdCol),
+                    nullability = nullability,
+                    constraints = constraints.getOrElse(coord, Nil)
+                  )
+                  (dbCol, mdCol.parsedColumnName)
+                }
 
-            db.View(relationName, tableCommentsByTable.get(relationName), decomposedSql, cols, deps, isMaterialized = viewRow.relkind == "m")
-          }
-          (relationName, lazyAnalysis)
+              case MaybeReturnsRows.Update => ???
+            }
+
+          db.View(relationName, tableCommentsByTable.get(relationName), decomposedSql, cols, deps, isMaterialized = viewRow.relkind == "m")
         }
-      }.toMap
+        (relationName, lazyAnalysis)
+      }
 
     val tables: Map[db.RelationName, Lazy[db.Table]] =
       input.tables.flatMap { relation =>