Merge remote-tracking branch 'origin/main' into feat/registry-file

README.md

@@ -1,6 +1,6 @@
 # Arke
 
-![Arke](https://user-images.githubusercontent.com/81776297/233303068-e112f6ec-98d8-426e-ab40-b887efe20db6.png)
+![Arke](https://github.com/arkemishub/arke/assets/81776297/7a04d11b-5cd0-4349-8621-d19cf0274585)
 
 ## Installation
 

lib/arke/core/unit.ex

@@ -262,6 +262,41 @@ defmodule Arke.Core.Unit do
 
   defp update_encoded_unit_data(_, data, _), do: data
 
+  def as_args(arke, unit) do
+    [
+      id: handle_id(unit.id),
+      arke_id: Atom.to_string(unit.arke_id),
+      data: encode_unit_data(arke, unit.data),
+      metadata: %{},
+      inserted_at: NaiveDateTime.utc_now(),
+      updated_at: NaiveDateTime.utc_now()
+    ]
+  end
+
+  defp handle_id(id) when is_nil(id), do: UUID.uuid1()
+  defp handle_id(id) when is_atom(id), do: Atom.to_string(id)
+  defp handle_id(id) when is_binary(id), do: id
+  # TODO handle error
+  defp handle_id(id), do: id
+
+  def encode_unit_data(arke, data) do
+    Enum.reduce(data, %{}, fn {key, value}, new_map ->
+      parameter = ArkeManager.get_parameter(arke, key)
+      update_encoded_unit_data(parameter, new_map, value)
+    end)
+  end
+
+  defp update_encoded_unit_data(%{data: %{only_runtime: true}}, data, _), do: data
+
+  defp update_encoded_unit_data(%{id: id}, data, value),
+       do:
+         Map.put_new(data, Atom.to_string(id), %{
+           :value => value,
+           :datetime => Arke.DatetimeHandler.now(:datetime)
+         })
+
+  defp update_encoded_unit_data(_, data, _), do: data
+
   # Handle parameters
   @doc """
   Get the Unit data as a keyword list

lib/arke/system.ex

@@ -64,56 +64,78 @@ defmodule Arke.System do
       end
 
       defp import_units(arke, project, member, file, mode) do
-        [{:ok, ref}] = Xlsxir.multi_extract(file.path)
+        {:ok, ref} = Enum.at(Xlsxir.multi_extract(file.path), 0)
         all_units = get_all_units_for_import(project)
 
         file_as_list = Xlsxir.get_list(ref)
-        header = get_header_for_import(project, arke, file_as_list)
+
+        header_file = Enum.at(file_as_list, 0)
         rows = file_as_list |> List.delete_at(0)
 
+        header = get_header_for_import(project, arke, header_file) |> parse_haeder_for_import(header_file)
+
         {correct_units, error_units} = Enum.with_index(rows) |> Enum.reduce({[], []}, fn {row, index}, {correct_units, error_units} ->
           case Enum.filter(row, & !is_nil(&1)) do
             [] -> {correct_units, error_units}
             _ -> case load_units(project, arke, header, row, all_units, mode) do
                    {:error, args, errors} ->
                      m = Enum.reduce(header, %{}, fn {h, index}, acc ->
-                       acc = Map.put(acc, h, Enum.at(row, index))
-                     end) |> Map.put(:errors, errors)
+                       acc = Map.put(acc, h, parse_cell(Enum.at(row, index)))
+                     end) |> Map.put("errors", errors)
                      {correct_units, [m | error_units]}
                    {:ok, unit_args} -> {[unit_args | correct_units], error_units}
                  end
           end
         end)
 
         existing_units = get_existing_units_for_import(project, arke, header, correct_units)
-        units_args = Enum.reduce(correct_units, [], fn u, units_args ->
-          case check_existing_units_for_import(project, arke, header, u, existing_units) do
-            true -> units_args
-            false -> [u | units_args]
-          end
-        end)
+        units_args = Enum.filter(correct_units, fn u -> check_existing_units_for_import(project, arke, header, u, existing_units) == false end)
+        if length(units_args) > 0 do
+          Enum.map(Stream.chunk_every(units_args, 5000) |> Enum.to_list(), fn chunk ->
+            ArkePostgres.Repo.insert_all("arke_unit", chunk, prefix: Atom.to_string(project))
+          end)
+        end
 
-        Enum.map(Stream.chunk_every(units_args, 5000) |> Enum.to_list(), fn chunk ->
-          ArkePostgres.Repo.insert_all("arke_unit", chunk, prefix: Atom.to_string(project))
-        end)
+        count_inserted =  length(units_args)
+        count_existing =  length(existing_units)
+        count_error =  length(error_units)
+        total_count =  count_inserted + count_error + count_existing
 
         res = %{
-          count_inserted: length(units_args),
-          count_existing: length(existing_units),
-          count_error: length(error_units),
+          count_inserted: count_inserted,
+          count_existing: count_existing,
+          count_error: count_error,
+          total_count: total_count,
           error_units: error_units
         }
         {:ok, res, 201}
       end
 
-      defp get_header_for_import(project, arke, file_as_list) do
-        Enum.reduce(Enum.with_index(Enum.at(file_as_list, 0)), [], fn {cell, index}, acc ->
+      defp parse_cell(value) when is_tuple(value), do: Kernel.inspect(value)
+      defp parse_cell(value), do: value
+
+      defp get_header_for_import(project, arke, header_file) do
+        Enum.reduce(Enum.with_index(header_file), [], fn {cell, index}, acc ->
           case Arke.Boundary.ArkeManager.get_parameter(arke, project, cell) do
             nil -> acc
-            parameter -> [{Atom.to_string(parameter.id), index} | acc]
+            parameter -> [Atom.to_string(parameter.id) | acc]
           end
         end)
       end
+      defp parse_haeder_for_import(header, header_file) do
+        Enum.reduce(Enum.with_index(header_file), [], fn {cell, index}, acc ->
+          case cell do
+            nil -> acc
+            "" -> acc
+            cell ->
+              case cell in header do
+                nil -> acc
+                parameter -> [{cell, index} | acc]
+              end
+          end
+        end)
+      end
+
       defp get_all_units_for_import(project), do: []
 
       defp load_units(project, arke, header, row, _, "default") do
@@ -246,20 +268,6 @@ defmodule Arke.System do
 
   defp get_base_arke_parameters(_type), do: nil
 
-  #  @spec __arke_info__(caller :: caller(), options :: list()) :: [id: atom() | String.t(), label: String.t(), active: boolean(), metadata: map(), type: atom()]
-  #  defp __arke_info__(caller, options) do
-  #
-  #    id = Keyword.get(options, :id, caller |> to_string |> String.split(".") |> List.last |> Macro.underscore |> String.to_atom)
-  #    label = Keyword.get(options, :label, id |> Atom.to_string |> String.replace("_", " ") |> String.capitalize)
-  #    [
-  #      id: id,
-  #      label: label,
-  #      active: Keyword.get(options, :active, true),
-  #      metadata: Keyword.get(options, :metadata, %{}),
-  #      type: Keyword.get(options, :type, :arke)
-  #    ]
-  #  end
-
   ######################################################################################################################
   # END ARKE MACRO #####################################################################################################
   ######################################################################################################################

mix.exs

@@ -1,7 +1,7 @@
 defmodule Arke.MixProject do
   use Mix.Project
 
-  @version "0.1.31"
+  @version "0.1.33"
   @scm_url "https://github.com/arkemishub/arke"
   @site_url "https://arkehub.com"