From 17f92d1e83a3613afca66fbc6a882cbaf8f5eac0 Mon Sep 17 00:00:00 2001 From: Gaurav Pareek Date: Tue, 23 Jan 2024 15:36:15 +0530 Subject: [PATCH 1/3] Created Start page --- apps/csv2sql/lib/csv2sql/config/loader.ex | 14 ++-- .../lib/dashboard_web/live/main_live.ex | 81 +++++++++++++++---- .../lib/dashboard_web/live/start_live.ex | 43 ++++++++++ 3 files changed, 118 insertions(+), 20 deletions(-) create mode 100644 apps/dashboard/lib/dashboard_web/live/start_live.ex diff --git a/apps/csv2sql/lib/csv2sql/config/loader.ex b/apps/csv2sql/lib/csv2sql/config/loader.ex index 4c76a4f..08b0a07 100644 --- a/apps/csv2sql/lib/csv2sql/config/loader.ex +++ b/apps/csv2sql/lib/csv2sql/config/loader.ex @@ -107,12 +107,16 @@ defmodule Csv2sql.Config.Loader do end defp load_db_config(args) do - db_type = get_db_type(args) + db_type = args[:db_type] + db_url = args[:db_url] + # db_type = get_db_type(args) - db_url = - if args[:db_url], - do: "ecto://#{args[:db_url]}", - else: raise("Please provide a valid database url") + # db_type = args[:db_type] + + # db_url = args[:db_url] + # if args[:db_url], + # do: "ecto://#{args[:db_url]}", + # else: raise("Please provide a valid database url") varchar_limit = get_varchar_limit(args) insertion_chunk_size = get_insertion_chunk_size(args) diff --git a/apps/dashboard/lib/dashboard_web/live/main_live.ex b/apps/dashboard/lib/dashboard_web/live/main_live.ex index 808de4f..f63161a 100644 --- a/apps/dashboard/lib/dashboard_web/live/main_live.ex +++ b/apps/dashboard/lib/dashboard_web/live/main_live.ex @@ -5,6 +5,7 @@ defmodule DashboardWeb.Live.MainLive do alias DashBoard.DbAttribute alias Csv2sql.Database.ConnectionTest alias DashboardWeb.Live.ConfigLive + alias DashboardWeb.Live.StartLive @debounce_time 1000 @@ -17,18 +18,50 @@ defmodule DashboardWeb.Live.MainLive do # Check for DB connection on config load from local storage timer_ref = Process.send_after(self(), :check_db_connection, @debounce_time) + IO.inspect(Csv2sql.ProgressTracker.get_state()) + + case Csv2sql.ProgressTracker.get_state().status do + {:error, %{message: message}} -> {:ok, + assign(socket, + page: "start", + modal: false, + path_validator_debouncer: nil, + db_connection_debouncer: timer_ref, + db_connection_established: false, + changeset: Config.get_defaults() |> Map.merge(local_storage_config) |> Config.changeset(), + matching_date_time: nil, + constraints: Csv2sql.Config.Loader.get_constraints(), + error: true, + reason: String.split(message, "\n") |> Enum.filter(fn s -> s not in ["",nil] end), + state: Csv2sql.ProgressTracker.get_state() + )} + + _ -> {:ok, + assign(socket, + page: "start", + modal: false, + path_validator_debouncer: nil, + db_connection_debouncer: timer_ref, + db_connection_established: false, + changeset: Config.get_defaults() |> Map.merge(local_storage_config) |> Config.changeset(), + matching_date_time: nil, + constraints: Csv2sql.Config.Loader.get_constraints(), + state: Csv2sql.ProgressTracker.get_state() + )} + end + end - {:ok, - assign(socket, - page: "config", - modal: false, - path_validator_debouncer: nil, - db_connection_debouncer: timer_ref, - db_connection_established: false, - changeset: Config.get_defaults() |> Map.merge(local_storage_config) |> Config.changeset(), - matching_date_time: nil, - constraints: Csv2sql.Config.Loader.get_constraints() - )} + def handle_event("csv-parse", _attrs, socket) do + try do + Csv2sql.Stages.Analyze.analyze_files() + IO.inspect(Csv2sql.ProgressTracker.get_state()) + Process.send_after(self(), :get_status, 2000) + {:noreply, assign(socket,page: "start",state: Csv2sql.ProgressTracker.get_state())} + catch + err -> + IO.inspect(err) + {:noreply, assign(socket,page: "start",error: true,err: err)} + end end @impl true @@ -115,6 +148,24 @@ defmodule DashboardWeb.Live.MainLive do |> update_matching_date_time()} end + @impl true + def handle_info(:get_status, socket) do + IO.inspect(Csv2sql.ProgressTracker.get_state().status) + case Csv2sql.ProgressTracker.get_state().status do + :finish -> + {:noreply, assign(socket,page: "start",state: Csv2sql.ProgressTracker.get_state())} + + {:error, %{message: message}} -> {:ok, + assign(socket, + page: "start", + error: true, + reason: String.split(message, "\n") |> Enum.filter(fn s -> s not in ["",nil] end), + state: Csv2sql.ProgressTracker.get_state() + )} + _ -> + Process.send_after(self(), :get_status, 2000) + end + end @impl true def handle_info(:check_db_connection, ~M{assigns} = socket) do with( @@ -161,10 +212,10 @@ defmodule DashboardWeb.Live.MainLive do ConfigLive.config_page(assigns) "start" -> - ~H""" - Placeholder - """ - + IO.inspect(assigns.changeset.changes.source_directory) + Map.put(assigns.changeset.changes, :dashboard, true) + Csv2sql.Config.Loader.load(Map.put(assigns.changeset.changes,:db_url, create_db_url(assigns.changeset.changes, false))) + StartLive.start_parse(assigns) "about" -> ~H""" About diff --git a/apps/dashboard/lib/dashboard_web/live/start_live.ex b/apps/dashboard/lib/dashboard_web/live/start_live.ex new file mode 100644 index 0000000..ba68e2c --- /dev/null +++ b/apps/dashboard/lib/dashboard_web/live/start_live.ex @@ -0,0 +1,43 @@ +defmodule DashboardWeb.Live.StartLive do + use DashboardWeb, :component + def start_parse(assigns) do + + ~H""" +

Start Parse

+
+ +
+ <%= case Map.get(assigns, :state).status do %> + <% :init -> %> +

Click on Start to parse CSVs

+ <% :working -> %> +

Analyzing files it may take some time

+ <% :finish -> %> +

The csv sucessfully parsed

+ <% _ -> %> +

There was a problem while parsing

+ + <% end %> + <%= if Map.get(assigns, :state).status != :init do%> + + + + + + + <%= Enum.map(Map.get(assigns, :state).files, fn {_path, file} -> %> + + + + + + <% end) %> +
File name rows_processed row_count
<%= file.name %> <%= file.rows_processed %> <%= file.row_count %>
+ <% end %> + """ + end +end From de01da310d38e16ff4f0a58cf1c2186b57f2e4c1 Mon Sep 17 00:00:00 2001 From: Gaurav Pareek Date: Wed, 24 Jan 2024 12:55:05 +0530 Subject: [PATCH 2/3] created start page to parse csvs --- .../lib/dashboard_web/live/main_live.ex | 99 +++++++++++-------- .../lib/dashboard_web/live/start_live.ex | 2 +- 2 files changed, 59 insertions(+), 42 deletions(-) diff --git a/apps/dashboard/lib/dashboard_web/live/main_live.ex b/apps/dashboard/lib/dashboard_web/live/main_live.ex index f63161a..70a83c8 100644 --- a/apps/dashboard/lib/dashboard_web/live/main_live.ex +++ b/apps/dashboard/lib/dashboard_web/live/main_live.ex @@ -21,33 +21,37 @@ defmodule DashboardWeb.Live.MainLive do IO.inspect(Csv2sql.ProgressTracker.get_state()) case Csv2sql.ProgressTracker.get_state().status do - {:error, %{message: message}} -> {:ok, - assign(socket, - page: "start", - modal: false, - path_validator_debouncer: nil, - db_connection_debouncer: timer_ref, - db_connection_established: false, - changeset: Config.get_defaults() |> Map.merge(local_storage_config) |> Config.changeset(), - matching_date_time: nil, - constraints: Csv2sql.Config.Loader.get_constraints(), - error: true, - reason: String.split(message, "\n") |> Enum.filter(fn s -> s not in ["",nil] end), - state: Csv2sql.ProgressTracker.get_state() - )} - - _ -> {:ok, - assign(socket, - page: "start", - modal: false, - path_validator_debouncer: nil, - db_connection_debouncer: timer_ref, - db_connection_established: false, - changeset: Config.get_defaults() |> Map.merge(local_storage_config) |> Config.changeset(), - matching_date_time: nil, - constraints: Csv2sql.Config.Loader.get_constraints(), - state: Csv2sql.ProgressTracker.get_state() - )} + {:error, %{message: message}} -> + {:ok, + assign(socket, + page: "start", + modal: false, + path_validator_debouncer: nil, + db_connection_debouncer: timer_ref, + db_connection_established: false, + changeset: + Config.get_defaults() |> Map.merge(local_storage_config) |> Config.changeset(), + matching_date_time: nil, + constraints: Csv2sql.Config.Loader.get_constraints(), + error: true, + reason: String.split(message, "\n") |> Enum.filter(fn s -> s not in ["", nil] end), + state: Csv2sql.ProgressTracker.get_state() + )} + + _ -> + {:ok, + assign(socket, + page: "start", + modal: false, + path_validator_debouncer: nil, + db_connection_debouncer: timer_ref, + db_connection_established: false, + changeset: + Config.get_defaults() |> Map.merge(local_storage_config) |> Config.changeset(), + matching_date_time: nil, + constraints: Csv2sql.Config.Loader.get_constraints(), + state: Csv2sql.ProgressTracker.get_state() + )} end end @@ -56,11 +60,11 @@ defmodule DashboardWeb.Live.MainLive do Csv2sql.Stages.Analyze.analyze_files() IO.inspect(Csv2sql.ProgressTracker.get_state()) Process.send_after(self(), :get_status, 2000) - {:noreply, assign(socket,page: "start",state: Csv2sql.ProgressTracker.get_state())} + {:noreply, assign(socket, page: "start", state: Csv2sql.ProgressTracker.get_state())} catch err -> IO.inspect(err) - {:noreply, assign(socket,page: "start",error: true,err: err)} + {:noreply, assign(socket, page: "start", error: true, err: err)} end end @@ -151,21 +155,25 @@ defmodule DashboardWeb.Live.MainLive do @impl true def handle_info(:get_status, socket) do IO.inspect(Csv2sql.ProgressTracker.get_state().status) + case Csv2sql.ProgressTracker.get_state().status do :finish -> - {:noreply, assign(socket,page: "start",state: Csv2sql.ProgressTracker.get_state())} - - {:error, %{message: message}} -> {:ok, - assign(socket, - page: "start", - error: true, - reason: String.split(message, "\n") |> Enum.filter(fn s -> s not in ["",nil] end), - state: Csv2sql.ProgressTracker.get_state() - )} - _ -> - Process.send_after(self(), :get_status, 2000) + {:noreply, assign(socket, page: "start", state: Csv2sql.ProgressTracker.get_state())} + + {:error, %{message: message}} -> + {:ok, + assign(socket, + page: "start", + error: true, + reason: String.split(message, "\n") |> Enum.filter(fn s -> s not in ["", nil] end), + state: Csv2sql.ProgressTracker.get_state() + )} + + _ -> + Process.send_after(self(), :get_status, 2000) end end + @impl true def handle_info(:check_db_connection, ~M{assigns} = socket) do with( @@ -214,8 +222,17 @@ defmodule DashboardWeb.Live.MainLive do "start" -> IO.inspect(assigns.changeset.changes.source_directory) Map.put(assigns.changeset.changes, :dashboard, true) - Csv2sql.Config.Loader.load(Map.put(assigns.changeset.changes,:db_url, create_db_url(assigns.changeset.changes, false))) + + Csv2sql.Config.Loader.load( + Map.put( + assigns.changeset.changes, + :db_url, + create_db_url(assigns.changeset.changes, false) + ) + ) + StartLive.start_parse(assigns) + "about" -> ~H""" About diff --git a/apps/dashboard/lib/dashboard_web/live/start_live.ex b/apps/dashboard/lib/dashboard_web/live/start_live.ex index ba68e2c..efcf3f4 100644 --- a/apps/dashboard/lib/dashboard_web/live/start_live.ex +++ b/apps/dashboard/lib/dashboard_web/live/start_live.ex @@ -1,7 +1,7 @@ defmodule DashboardWeb.Live.StartLive do use DashboardWeb, :component - def start_parse(assigns) do + def start_parse(assigns) do ~H"""

Start Parse

From 2a89c947a2747101a8ff2a2ee928f81ac94608f4 Mon Sep 17 00:00:00 2001 From: Gaurav Pareek Date: Wed, 24 Jan 2024 15:46:55 +0530 Subject: [PATCH 3/3] formatted code --- apps/dashboard/lib/dashboard_web/live/main_live.ex | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/apps/dashboard/lib/dashboard_web/live/main_live.ex b/apps/dashboard/lib/dashboard_web/live/main_live.ex index 70a83c8..c9499f1 100644 --- a/apps/dashboard/lib/dashboard_web/live/main_live.ex +++ b/apps/dashboard/lib/dashboard_web/live/main_live.ex @@ -18,7 +18,6 @@ defmodule DashboardWeb.Live.MainLive do # Check for DB connection on config load from local storage timer_ref = Process.send_after(self(), :check_db_connection, @debounce_time) - IO.inspect(Csv2sql.ProgressTracker.get_state()) case Csv2sql.ProgressTracker.get_state().status do {:error, %{message: message}} -> @@ -58,13 +57,11 @@ defmodule DashboardWeb.Live.MainLive do def handle_event("csv-parse", _attrs, socket) do try do Csv2sql.Stages.Analyze.analyze_files() - IO.inspect(Csv2sql.ProgressTracker.get_state()) Process.send_after(self(), :get_status, 2000) {:noreply, assign(socket, page: "start", state: Csv2sql.ProgressTracker.get_state())} catch err -> - IO.inspect(err) - {:noreply, assign(socket, page: "start", error: true, err: err)} + {:noreply, assign(socket, page: "start", error: true)} end end @@ -220,7 +217,6 @@ defmodule DashboardWeb.Live.MainLive do ConfigLive.config_page(assigns) "start" -> - IO.inspect(assigns.changeset.changes.source_directory) Map.put(assigns.changeset.changes, :dashboard, true) Csv2sql.Config.Loader.load(