fix: more review comments

projects/extension/sql/idempotent/013-loading.sql

@@ -1,7 +1,7 @@
 -------------------------------------------------------------------------------
 -- loading_column
 create or replace function ai.loading_column
-( column_name pg_catalog.text)
+( column_name pg_catalog.name)
 returns pg_catalog.jsonb
 as $func$
     select json_object
@@ -16,7 +16,7 @@ set search_path to pg_catalog, pg_temp
 -------------------------------------------------------------------------------
 -- loading_uri
 create or replace function ai.loading_uri
-( column_name pg_catalog.text)
+( column_name pg_catalog.name)
 returns pg_catalog.jsonb
 as $func$
     select json_object
@@ -39,7 +39,7 @@ as $func$
 declare
     _config_type pg_catalog.text;
     _implementation pg_catalog.text;
-    _column_name pg_catalog.text;
+    _column_name pg_catalog.name;
     _found pg_catalog.bool;
 begin
     if pg_catalog.jsonb_typeof(config) operator(pg_catalog.!=) 'object' then

projects/extension/sql/idempotent/014-parsing.sql

@@ -56,7 +56,7 @@ create or replace function ai._validate_parsing
 ) returns void
 as $func$
 declare
-    _column_type pg_catalog.text;
+    _column_type pg_catalog.name;
     _config_type pg_catalog.text;
     _loading_implementation pg_catalog.text;
     _parsing_implementation pg_catalog.text;

projects/extension/sql/idempotent/900-semantic-catalog-init.sql

@@ -39,6 +39,7 @@ begin
     , grant_to=>grant_to
     , formatting=>ai.formatting_python_template() -- TODO: this ain't gonna work
     , chunking=>ai.chunking_recursive_character_text_splitter() -- TODO
+    , loading=>ai.loading_column('description')
     ) into strict _obj_vec_id
     ;
 
@@ -52,6 +53,7 @@ begin
     , grant_to=>grant_to
     , formatting=>ai.formatting_python_template() -- TODO: this ain't gonna work
     , chunking=>ai.chunking_recursive_character_text_splitter() -- TODO
+    , loading=>ai.loading_column('description')
     ) into strict _sql_vec_id
     ;
 

projects/extension/tests/contents/output16.expected

@@ -42,8 +42,8 @@ CREATE EXTENSION
  function ai.litellm_embed(text,text[],text,text,jsonb,boolean)
  function ai.load_dataset_multi_txn(text,text,text,name,name,text,jsonb,integer,integer,integer,jsonb)
  function ai.load_dataset(text,text,text,name,name,text,jsonb,integer,integer,jsonb)
- function ai.loading_column(text)
- function ai.loading_uri(text)
+ function ai.loading_column(name)
+ function ai.loading_uri(name)
  function ai.ollama_chat_complete(text,jsonb,text,text,jsonb,jsonb,jsonb,boolean)
  function ai.ollama_embed(text,text,text,text,jsonb,boolean)
  function ai.ollama_generate(text,text,text,bytea[],text,jsonb,text,text,integer[],boolean)

projects/extension/tests/contents/output17.expected

@@ -42,8 +42,8 @@ CREATE EXTENSION
  function ai.litellm_embed(text,text[],text,text,jsonb,boolean)
  function ai.load_dataset_multi_txn(text,text,text,name,name,text,jsonb,integer,integer,integer,jsonb)
  function ai.load_dataset(text,text,text,name,name,text,jsonb,integer,integer,jsonb)
- function ai.loading_column(text)
- function ai.loading_uri(text)
+ function ai.loading_column(name)
+ function ai.loading_uri(name)
  function ai.ollama_chat_complete(text,jsonb,text,text,jsonb,jsonb,jsonb,boolean)
  function ai.ollama_embed(text,text,text,text,jsonb,boolean)
  function ai.ollama_generate(text,text,text,bytea[],text,jsonb,text,text,integer[],boolean)

projects/extension/tests/privileges/function.expected

@@ -264,14 +264,14 @@
  p       | bob   | execute   | no      | ai     | load_dataset_multi_txn(IN name text, IN config_name text, IN split text, IN schema_name name, IN table_name name, IN if_table_exists text, IN field_types jsonb, IN batch_size integer, IN max_batches integer, IN commit_every_n_batches integer, IN kwargs jsonb)
  p       | fred  | execute   | no      | ai     | load_dataset_multi_txn(IN name text, IN config_name text, IN split text, IN schema_name name, IN table_name name, IN if_table_exists text, IN field_types jsonb, IN batch_size integer, IN max_batches integer, IN commit_every_n_batches integer, IN kwargs jsonb)
  p       | jill  | execute   | YES     | ai     | load_dataset_multi_txn(IN name text, IN config_name text, IN split text, IN schema_name name, IN table_name name, IN if_table_exists text, IN field_types jsonb, IN batch_size integer, IN max_batches integer, IN commit_every_n_batches integer, IN kwargs jsonb)
- f       | alice | execute   | YES     | ai     | loading_column(column_name text)
- f       | bob   | execute   | no      | ai     | loading_column(column_name text)
- f       | fred  | execute   | no      | ai     | loading_column(column_name text)
- f       | jill  | execute   | YES     | ai     | loading_column(column_name text)
- f       | alice | execute   | YES     | ai     | loading_uri(column_name text)
- f       | bob   | execute   | no      | ai     | loading_uri(column_name text)
- f       | fred  | execute   | no      | ai     | loading_uri(column_name text)
- f       | jill  | execute   | YES     | ai     | loading_uri(column_name text)
+ f       | alice | execute   | YES     | ai     | loading_column(column_name name)
+ f       | bob   | execute   | no      | ai     | loading_column(column_name name)
+ f       | fred  | execute   | no      | ai     | loading_column(column_name name)
+ f       | jill  | execute   | YES     | ai     | loading_column(column_name name)
+ f       | alice | execute   | YES     | ai     | loading_uri(column_name name)
+ f       | bob   | execute   | no      | ai     | loading_uri(column_name name)
+ f       | fred  | execute   | no      | ai     | loading_uri(column_name name)
+ f       | jill  | execute   | YES     | ai     | loading_uri(column_name name)
  f       | alice | execute   | YES     | ai     | ollama_chat_complete(model text, messages jsonb, host text, keep_alive text, chat_options jsonb, tools jsonb, response_format jsonb, "verbose" boolean)
  f       | bob   | execute   | no      | ai     | ollama_chat_complete(model text, messages jsonb, host text, keep_alive text, chat_options jsonb, tools jsonb, response_format jsonb, "verbose" boolean)
  f       | fred  | execute   | no      | ai     | ollama_chat_complete(model text, messages jsonb, host text, keep_alive text, chat_options jsonb, tools jsonb, response_format jsonb, "verbose" boolean)

projects/extension/tests/vectorizer/server.py

@@ -108,6 +108,38 @@ class ProcessingDefault(BaseModel):
     concurrency: int | None = None
 
 
+class LoadingColumn(BaseModel):
+    implementation: Literal["row"]
+    config_type: Literal["loading"]
+    column_name: str
+
+
+class LoadingUri(BaseModel):
+    implementation: Literal["document"]
+    config_type: Literal["loading"]
+    column_name: str
+
+
+class ParsingAuto(BaseModel):
+    implementation: Literal["auto"]
+    config_type: Literal["parsing"]
+
+
+class ParsingNone(BaseModel):
+    implementation: Literal["none"]
+    config_type: Literal["parsing"]
+
+
+class ParsingPyMuPDF(BaseModel):
+    implementation: Literal["pymupdf"]
+    config_type: Literal["parsing"]
+
+
+class ParsingDocling(BaseModel):
+    implementation: Literal["docling"]
+    config_type: Literal["parsing"]
+
+
 class Config(BaseModel):
     version: str
     indexing: IndexingNone | IndexingDiskANN | IndexingHNSW = Field(
@@ -122,6 +154,8 @@ class Config(BaseModel):
         Field(..., discriminator="implementation")
     )
     processing: ProcessingDefault = Field(..., discriminator="implementation")
+    loading: LoadingColumn | LoadingUri = Field(..., discriminator="implementation")
+    parsing: ParsingAuto | ParsingNone | ParsingPyMuPDF | ParsingDocling = Field(..., discriminator="implementation")
 
 
 class PrimaryKeyColumn(BaseModel):

projects/extension/tests/vectorizer/test_loading.py

@@ -83,6 +83,27 @@ def test_validate_loading():
             """,
             "function ai.loading_uri() does not exist",
         ),
+        (
+            """
+            select ai._validate_loading
+            ( ai.loading_column('column_does_not_exist'), 'public', 'thing' )
+            """,
+            "column_name in config does not exist in the table: column_does_not_exist"
+        ),
+        (
+            """
+            select ai._validate_loading
+            ( ai.loading_column('weight'), 'public', 'thing' )
+            """,
+            "column_name in config does not exist in the table: weight"
+        ),
+        (
+            """
+            select ai._validate_loading
+            ( ai.loading_uri('weight'), 'public', 'thing' )
+            """,
+            "column_name in config does not exist in the table: weight"
+        ),
         (
             """
             select ai._validate_loading