[Refactor] Reorganised code structure (cont'd)

rte-france · Dec 17, 2024 · fab7e83 · fab7e83
1 parent bcb1f6e
commit fab7e83
Show file tree

Hide file tree

Showing 16 changed files with 37 additions and 27 deletions.
diff --git a/bertrend/summary/start_summarizer_app.sh → bertrend/demos/start_summarizer_app.sh b/bertrend/summary/start_summarizer_app.sh → bertrend/demos/start_summarizer_app.sh
@@ -5,4 +5,4 @@
 # This file is part of BERTrend.
 #
 
-CUDA_VISIBLE_DEVICES=0 OPENAI_API_KEY=$AZURE_WATTELSE_OPENAI_API_KEY OPENAI_ENDPOINT=$AZURE_WATTELSE_OPENAI_ENDPOINT OPENAI_DEFAULT_MODEL_NAME=$AZURE_WATTELSE_OPENAI_DEFAULT_MODEL_NAME streamlit run --theme.primaryColor royalblue app.py
+CUDA_VISIBLE_DEVICES=0 OPENAI_API_KEY=$AZURE_WATTELSE_OPENAI_API_KEY OPENAI_ENDPOINT=$AZURE_WATTELSE_OPENAI_ENDPOINT OPENAI_DEFAULT_MODEL_NAME=$AZURE_WATTELSE_OPENAI_DEFAULT_MODEL_NAME streamlit run --theme.primaryColor royalblue summarization/summarizer_app.py
diff --git a/bertrend/demos/start_topic_analysis_app.sh b/bertrend/demos/start_topic_analysis_app.sh
@@ -0,0 +1,8 @@
+#
+# Copyright (c) 2024, RTE (https://www.rte-france.com)
+# See AUTHORS.txt
+# SPDX-License-Identifier: MPL-2.0
+# This file is part of BERTrend.
+#
+
+CUDA_VISIBLE_DEVICES=0 OPENAI_API_KEY=$AZURE_WATTELSE_OPENAI_API_KEY OPENAI_ENDPOINT=$AZURE_WATTELSE_OPENAI_ENDPOINT OPENAI_DEFAULT_MODEL_NAME=$AZURE_WATTELSE_OPENAI_DEFAULT_MODEL_NAME streamlit run --theme.primaryColor royalblue topic_analysis/Main_page.py
diff --git a/bertrend/demos/start_weak_signals_app.sh b/bertrend/demos/start_weak_signals_app.sh
@@ -0,0 +1,8 @@
+#
+# Copyright (c) 2024, RTE (https://www.rte-france.com)
+# See AUTHORS.txt
+# SPDX-License-Identifier: MPL-2.0
+# This file is part of BERTrend.
+#
+
+CUDA_VISIBLE_DEVICES=0 OPENAI_API_KEY=$AZURE_WATTELSE_OPENAI_API_KEY OPENAI_ENDPOINT=$AZURE_WATTELSE_OPENAI_ENDPOINT OPENAI_DEFAULT_MODEL_NAME=$AZURE_WATTELSE_OPENAI_DEFAULT_MODEL_NAME streamlit run --theme.primaryColor royalblue weak_signals/app.py
diff --git a/bertrend/demos/summarization/summarizer_app.py b/bertrend/demos/summarization/summarizer_app.py
@@ -7,9 +7,9 @@
 import streamlit as st
 from loguru import logger
 
-from bertrend.summary.abstractive_summarizer import AbstractiveSummarizer
-from bertrend.summary.chatgpt_summarizer import GPTSummarizer
-from bertrend.summary.extractive_summarizer import (
+from bertrend.services.summary.abstractive_summarizer import AbstractiveSummarizer
+from bertrend.services.summary.chatgpt_summarizer import GPTSummarizer
+from bertrend.services.summary.extractive_summarizer import (
     EnhancedExtractiveSummarizer,
     ExtractiveSummarizer,
 )

diff --git a/bertrend/demos/topic_analysis/pages/4_Generate_Newsletters.py b/bertrend/demos/topic_analysis/pages/4_Generate_Newsletters.py
@@ -12,9 +12,9 @@
     register_widget,
     save_widget_state,
 )
-from bertrend.summary import GPTSummarizer
-from bertrend.summary.abstractive_summarizer import AbstractiveSummarizer
-from bertrend.summary.extractive_summarizer import (
+from bertrend.services.summary import GPTSummarizer
+from bertrend.services.summary.abstractive_summarizer import AbstractiveSummarizer
+from bertrend.services.summary.extractive_summarizer import (
     ExtractiveSummarizer,
     EnhancedExtractiveSummarizer,
 )

diff --git a/bertrend/summary/summarizer.py → bertrend/services/summarizer.py b/bertrend/summary/summarizer.py → bertrend/services/summarizer.py
diff --git a/bertrend/summary/__init__.py → bertrend/services/summary/__init__.py b/bertrend/summary/__init__.py → bertrend/services/summary/__init__.py
@@ -2,5 +2,3 @@
 #  See AUTHORS.txt
 #  SPDX-License-Identifier: MPL-2.0
 #  This file is part of BERTrend.
-
-from bertrend.summary.chatgpt_summarizer import GPTSummarizer
diff --git a/bertrend/summary/abstractive_summarizer.py → ...ervices/summary/abstractive_summarizer.py b/bertrend/summary/abstractive_summarizer.py → ...ervices/summary/abstractive_summarizer.py
@@ -9,7 +9,7 @@
 import torch
 from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
 
-from bertrend.summary.summarizer import Summarizer
+from bertrend.services.summarizer import Summarizer
 
 device = "cuda:0" if torch.cuda.is_available() else "cpu"
 

diff --git a/bertrend/summary/chatgpt_summarizer.py → ...nd/services/summary/chatgpt_summarizer.py b/bertrend/summary/chatgpt_summarizer.py → ...nd/services/summary/chatgpt_summarizer.py
@@ -6,13 +6,13 @@
 from loguru import logger
 
 from bertrend.llm_utils.openai_client import OpenAI_Client
-from bertrend.summary.prompts import (
+from bertrend.services.summary.prompts import (
     FR_SYSTEM_SUMMARY_SENTENCES,
     EN_SYSTEM_SUMMARY_SENTENCES,
 )
-from bertrend.summary.summarizer import DEFAULT_MAX_SENTENCES
+from bertrend.services.summarizer import DEFAULT_MAX_SENTENCES
 
-from bertrend.summary.summarizer import Summarizer
+from bertrend.services.summarizer import Summarizer
 
 
 class GPTSummarizer(Summarizer):

diff --git a/bertrend/summary/extractive_summarizer.py → ...services/summary/extractive_summarizer.py b/bertrend/summary/extractive_summarizer.py → ...services/summary/extractive_summarizer.py
@@ -14,12 +14,12 @@
 from torch import Tensor
 
 from bertrend.llm_utils.openai_client import OpenAI_Client
-from bertrend.summary.lexrank import degree_centrality_scores
-from bertrend.summary.prompts import (
+from bertrend.services.summary.lexrank import degree_centrality_scores
+from bertrend.services.summary.prompts import (
     FR_SYSTEM_SUMMARY_SENTENCES,
     EN_SYSTEM_SUMMARY_SENTENCES,
 )
-from bertrend.summary.summarizer import (
+from bertrend.services.summarizer import (
     Summarizer,
     DEFAULT_MAX_SENTENCES,
     DEFAULT_SUMMARIZATION_RATIO,

diff --git a/bertrend/summary/lexrank.py → bertrend/services/summary/lexrank.py b/bertrend/summary/lexrank.py → bertrend/services/summary/lexrank.py
diff --git a/bertrend/summary/prompts.py → bertrend/services/summary/prompts.py b/bertrend/summary/prompts.py → bertrend/services/summary/prompts.py
diff --git a/bertrend/trend_analysis/weak_signals.py b/bertrend/trend_analysis/weak_signals.py
@@ -25,7 +25,7 @@
     SIGNAL_CLASSIF_UPPER_BOUND,
 )
 from bertrend import SIGNAL_EVOLUTION_DATA_DIR
-from bertrend.signal_analysis.prompts import get_prompt, save_html_output
+from bertrend.trend_analysis.prompts import get_prompt, save_html_output
 
 
 def detect_weak_signals_zeroshot(

diff --git a/bertrend_apps/data_provider/arxiv_provider.py b/bertrend_apps/data_provider/arxiv_provider.py
@@ -14,7 +14,7 @@
 from loguru import logger
 import requests
 
-from bertrend.utils import TEXT_COLUMN
+from bertrend.utils.data_loading import TEXT_COLUMN
 from bertrend_apps.data_provider.data_provider import DataProvider
 from bertrend_apps.data_provider.utils import wait
 

diff --git a/bertrend_apps/exploration/curebot/veille_analyse.py b/bertrend_apps/exploration/curebot/veille_analyse.py
@@ -12,14 +12,10 @@
 from loguru import logger
 from streamlit.runtime.uploaded_file_manager import UploadedFile
 
-from bertrend.summary.chatgpt_summarizer import GPTSummarizer
+from bertrend.services.summary.chatgpt_summarizer import GPTSummarizer
 from bertrend.train import train_BERTopic
-from bertrend.utils import (
-    load_data,
-    TIMESTAMP_COLUMN,
-    split_df_by_paragraphs,
-    clean_dataset,
-)
+from bertrend.utils.data_loading import load_data, TIMESTAMP_COLUMN, split_df_by_paragraphs, clean_dataset
+
 from bertrend_apps.data_provider.curebot_provider import CurebotProvider
 from bertrend_apps.newsletters.newsletter_features import generate_newsletter, md2html
 

diff --git a/bertrend_apps/newsletters/newsletter_features.py b/bertrend_apps/newsletters/newsletter_features.py
@@ -21,8 +21,8 @@
     FR_USER_GENERATE_TOPIC_LABEL_SUMMARIES_V2,
     EN_USER_GENERATE_TOPIC_LABEL_SUMMARIES_V2,
 )
-from bertrend.summary.summarizer import Summarizer
-from bertrend.summary.abstractive_summarizer import AbstractiveSummarizer
+from bertrend.services.summarizer import Summarizer
+from bertrend.services.summary.abstractive_summarizer import AbstractiveSummarizer
 from bertopic._bertopic import BERTopic
 from tqdm import tqdm