[WIP] Adding support for deltalake file

lib/explorer/backend/data_frame.ex

@@ -79,6 +79,9 @@ defmodule Explorer.Backend.DataFrame do
               eol_delimiter :: option(String.t())
             ) :: io_result(df)
 
+  # IO: deltalake
+  @callback to_delta(df, table_uri :: fs_entry()) :: ok_result()
+
   # IO: Parquet
   @callback from_parquet(
               entry :: fs_entry(),

lib/explorer/data_frame.ex

@@ -1392,6 +1392,33 @@ defmodule Explorer.DataFrame do
     end
   end
 
+  @doc """
+  Writes a dataframe to a deltalake format file.
+  """
+  @doc tyoe: :io
+  @spec to_delta(df :: DataFrame.t(), table_uri :: String.t() | fs_entry()) :: :ok | {:error, Exception.t()}
+  def to_delta(df, table_uri) do
+    Shared.apply_impl(df, :to_delta, [table_uri])
+  end
+
+  @doc """
+  Similar to `to_delta/2`, but raises in case of error.
+  """
+  @doc type: :io
+  @spec to_delta!(df :: DataFrame.t(), table_uri :: String.t() | fs_entry()) :: :ok
+  def to_delta!(df, filename) do
+    case to_delta(df, filename) do
+      :ok ->
+        :ok
+
+      {:error, %module{} = e} when module in [ArgumentError, RuntimeError] ->
+        raise module, "to_delta failed: #{inspect(e.message)}"
+
+      {:error, error} ->
+        raise "to_delta failed: #{inspect(error)}"
+    end
+  end
+
   @doc """
   Read a file of JSON objects or lists separated by new lines
 

lib/explorer/polars_backend/data_frame.ex

@@ -222,6 +222,22 @@ defmodule Explorer.PolarsBackend.DataFrame do
   defp char_byte(nil), do: nil
   defp char_byte(<<char::utf8>>), do: char
 
+  @impl true
+  def to_delta(%DataFrame{data: df}, %Local.Entry{} = entry) do
+    case Native.df_to_delta(df, entry.path) do
+      {:ok, _} -> :ok
+      {:error, error} -> {:error, RuntimeError.exception(error)}
+    end
+  end
+
+  @impl true
+  def to_delta(%DataFrame{data: df}, entry) do
+    case Native.df_to_delta(df, entry) do
+      {:ok, _} -> :ok
+      {:error, error} -> {:error, RuntimeError.exception(error)}
+    end
+  end
+
   @impl true
 
   def from_ndjson(%module{} = entry, infer_schema_length, batch_size)

lib/explorer/polars_backend/native.ex

@@ -168,6 +168,7 @@ defmodule Explorer.PolarsBackend.Native do
   def df_transpose(_df, _keep_names_as, _new_col_names), do: err()
   def df_to_csv(_df, _filename, _has_headers, _delimiter), do: err()
   def df_to_csv_cloud(_df, _ex_entry, _has_headers, _delimiter), do: err()
+  def df_to_delta(_df, _table_uri), do: err()
   def df_to_dummies(_df, _columns), do: err()
   def df_to_ipc(_df, _filename, _compression), do: err()
   def df_to_ipc_cloud(_df, _ex_entry, _compression), do: err()