From 644e7ec0a560276c974b493f692b739da3ca943a Mon Sep 17 00:00:00 2001 From: Olivier DUVAL Date: Thu, 23 Oct 2025 12:33:45 +0200 Subject: [PATCH 01/13] refacto : Organiser le projet en modules (app/ et tests/) https://github.com/zorky/techno_watch_ia/issues/25 --- nodes/utils_fetch_nodes.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/nodes/utils_fetch_nodes.py b/nodes/utils_fetch_nodes.py index 93b896b..dcd8be8 100644 --- a/nodes/utils_fetch_nodes.py +++ b/nodes/utils_fetch_nodes.py @@ -96,11 +96,10 @@ def _load_sources_from_config(config_path: str, type_source: str) -> list[Source """ config = _load_json_reddit_bluesky(config_path) - - logger.info(Fore.LIGHTRED_EX + f"Source pour {type_source}") + + logger.info(Fore.LIGHTRED_EX + f"filter sources Reddit ou Bluesky : {type_source}") sources = [] - for source_config in config.get('sources', []): - logger.info(Fore.RED + f"filter sources Reddit ou Bluesky {type_source}") + for source_config in config.get('sources', []): if type_source == 'reddit': sources.append(Source( type=SourceType.REDDIT, From 8149daaf8c38d84d7dafaf3a994e92dfc2bf9ba0 Mon Sep 17 00:00:00 2001 From: Olivier DUVAL Date: Thu, 23 Oct 2025 14:00:01 +0200 Subject: [PATCH 02/13] =?UTF-8?q?refactoring=20complet=20:=20d=C3=A9placem?= =?UTF-8?q?ents=20des=20r=C3=A9pertoires=20et=20fix=20imports?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit app tests web.py --- app/__init__.py | 6 ++--- {core => app/core}/__init__.py | 0 {core => app/core}/logger.py | 7 +++--- {core => app/core}/utils.py | 8 +++---- app/main_agent.py | 23 ++++++++++--------- {nodes => app/nodes}/__init__.py | 0 {nodes => app/nodes}/fetch_nodes.py | 12 ++++++---- {nodes => app/nodes}/filter_nodes.py | 12 ++++++---- {nodes => app/nodes}/output_nodes.py | 2 +- {nodes => app/nodes}/save_nodes.py | 2 +- {nodes => app/nodes}/send_nodes.py | 9 ++++---- {nodes => app/nodes}/summarize_nodes.py | 18 +++++++-------- {nodes => app/nodes}/utils_fetch_nodes.py | 18 +++++++-------- {services => app/services}/decorators.py | 4 ++-- {services => app/services}/factory_fetcher.py | 6 ++--- .../services}/fetchers/__init__.py | 0 .../services}/fetchers/base_fetcher.py | 2 +- .../services}/fetchers/bluesky_fetcher.py | 15 ++++++------ .../services}/fetchers/reedit_fetcher.py | 16 ++++++------- .../services}/fetchers/rss_fetcher.py | 14 +++++------ {services => app/services}/model_service.py | 4 ++-- {services => app/services}/models.py | 0 .../services}/sources_ponderation.py | 14 +++++------ {services => app/services}/utils_fetchers.py | 8 ++++--- db/db.py | 2 +- jinja_filters.py | 2 +- models/article.py | 2 +- read_opml.py | 2 +- send_articles_email.py | 2 +- tests/conftest.py | 2 +- tests/test_sources_ponderation.py | 9 ++++---- web.py | 2 +- 32 files changed, 114 insertions(+), 109 deletions(-) rename {core => app/core}/__init__.py (100%) rename {core => app/core}/logger.py (97%) rename {core => app/core}/utils.py (88%) rename {nodes => app/nodes}/__init__.py (100%) rename {nodes => app/nodes}/fetch_nodes.py (94%) rename {nodes => app/nodes}/filter_nodes.py (93%) rename {nodes => app/nodes}/output_nodes.py (94%) rename {nodes => app/nodes}/save_nodes.py (91%) rename {nodes => app/nodes}/send_nodes.py (83%) rename {nodes => app/nodes}/summarize_nodes.py (88%) rename {nodes => app/nodes}/utils_fetch_nodes.py (89%) rename {services => app/services}/decorators.py (95%) rename {services => app/services}/factory_fetcher.py (94%) rename {services => app/services}/fetchers/__init__.py (100%) rename {services => app/services}/fetchers/base_fetcher.py (81%) rename {services => app/services}/fetchers/bluesky_fetcher.py (97%) rename {services => app/services}/fetchers/reedit_fetcher.py (90%) rename {services => app/services}/fetchers/rss_fetcher.py (93%) rename {services => app/services}/model_service.py (98%) rename {services => app/services}/models.py (100%) rename {services => app/services}/sources_ponderation.py (96%) rename {services => app/services}/utils_fetchers.py (91%) diff --git a/app/__init__.py b/app/__init__.py index c41b9a4..6202c0d 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -1,13 +1,13 @@ # from core import measure_time, argscli, logger -from core.logger import logger -from core.utils import measure_time #, get_environment_variable +from .core.logger import logger +from .core.utils import measure_time #, get_environment_variable from models.emails import EmailTemplateParams from models.states import RSSState __all__ = [ 'logger', 'measure_time', - 'argscli', + # 'argscli', 'EmailTemplateParams', 'RSSState' ] \ No newline at end of file diff --git a/core/__init__.py b/app/core/__init__.py similarity index 100% rename from core/__init__.py rename to app/core/__init__.py diff --git a/core/logger.py b/app/core/logger.py similarity index 97% rename from core/logger.py rename to app/core/logger.py index 4a822f8..0957a81 100644 --- a/core/logger.py +++ b/app/core/logger.py @@ -5,6 +5,9 @@ logging.basicConfig(level=logging.INFO) +def print_color(color, text): + print(color + text) + # ========================= # Formatter coloré # ========================= @@ -39,12 +42,8 @@ def setup_logger(level=logging.INFO): logger = setup_logger() -def print_color(color, text): - print(color + text) - def count_by_type_articles(title, articles_by_source, color=Fore.LIGHTYELLOW_EX): from collections import Counter - from core.logger import print_color source_counts = Counter(item['source'].value for item in articles_by_source) print_color(color, "=" * 60) print_color(color, f"{title} {source_counts}") diff --git a/core/utils.py b/app/core/utils.py similarity index 88% rename from core/utils.py rename to app/core/utils.py index d879393..14771dc 100644 --- a/core/utils.py +++ b/app/core/utils.py @@ -4,8 +4,9 @@ import time import logging import argparse -from colorama import Fore, Style, init -from core.logger import setup_logger + +from app.core.logger import setup_logger +from app.core.logger import logger # ========================= # Configuration du logging @@ -24,8 +25,7 @@ def configure_logging_from_args(): def measure_time(func): @wraps(func) - def wrapper(*args, **kwargs): - from core.logger import logger + def wrapper(*args, **kwargs): start_time = time.time() result = func(*args, **kwargs) end_time = time.time() diff --git a/app/main_agent.py b/app/main_agent.py index 8d826ac..9dc4ace 100644 --- a/app/main_agent.py +++ b/app/main_agent.py @@ -31,14 +31,19 @@ # from read_opml import parse_opml_to_rss_list # from core import argscli -from services.utils_fetchers import register_fetchers_auto -from services.models import SourceType, UnifiedState +from .services.utils_fetchers import register_fetchers_auto +from .services.models import SourceType, UnifiedState -from core import get_environment_variable -from core.logger import logger +from .core import get_environment_variable +from .core.logger import logger -from nodes import filter_node, summarize_node, \ +from .nodes import filter_node, summarize_node, \ output_node, save_articles_node, send_articles_node +from .nodes import dispatch_node, fetch_rss_node, fetch_reddit_node, fetch_bluesky_node, merge_fetched_articles + +from .core.utils import configure_logging_from_args + +from .core.logger import print_color # ========================= # Init du logging et logger @@ -152,9 +157,7 @@ def wrapper(state: UnifiedState) -> UnifiedState: # Construction du graphe : noeuds (nodes) et transitions (edges) # fetch -> filter -> summarize -> output # ========================= -def make_graph(): - from nodes import dispatch_node, fetch_rss_node, fetch_reddit_node, fetch_bluesky_node, merge_fetched_articles - from core.logger import print_color +def make_graph(): RSS_FETCH, REDDIT_FETCH, BLUESKY_FETCH = which_fetcher() fetcher_flags = { "RSS_FETCH": RSS_FETCH, @@ -276,9 +279,7 @@ def prepare_data(): # Main # ========================= def main(): - from db.db import init_db - from core.utils import configure_logging_from_args - # from core.logger import logger + from db.db import init_db _, args = configure_logging_from_args() logger.info(Fore.MAGENTA + Style.BRIGHT + "=== Agent RSS avec résumés LLM ===") diff --git a/nodes/__init__.py b/app/nodes/__init__.py similarity index 100% rename from nodes/__init__.py rename to app/nodes/__init__.py diff --git a/nodes/fetch_nodes.py b/app/nodes/fetch_nodes.py similarity index 94% rename from nodes/fetch_nodes.py rename to app/nodes/fetch_nodes.py index 32bce28..f4af74a 100644 --- a/nodes/fetch_nodes.py +++ b/app/nodes/fetch_nodes.py @@ -1,10 +1,12 @@ from functools import lru_cache -from services.factory_fetcher import FetcherFactory -from services.models import Source, SourceType, UnifiedState -from core.logger import logger from colorama import Fore import time -from core.utils import get_environment_variable + +from app.services.factory_fetcher import FetcherFactory +from app.services.models import Source, SourceType, UnifiedState +from app.core.logger import logger +from app.core.logger import print_color +from app.core.utils import get_environment_variable from .utils_fetch_nodes import fetch_articles, get_rss_urls, register_fetchers, get_bluesky_urls, get_subs_reddit_urls import logging @@ -83,7 +85,7 @@ def merge_fetched_articles(state: UnifiedState) -> dict: """Noeud de fusion des données ramenés par les fetchers""" """Fusionne tous les articles des différentes sources""" from collections import Counter - from core.logger import print_color + all_articles = [] diff --git a/nodes/filter_nodes.py b/app/nodes/filter_nodes.py similarity index 93% rename from nodes/filter_nodes.py rename to app/nodes/filter_nodes.py index c584805..54e3a65 100644 --- a/nodes/filter_nodes.py +++ b/app/nodes/filter_nodes.py @@ -3,9 +3,11 @@ logging.basicConfig(level=logging.INFO) from colorama import Fore -from core.logger import logger -from services.models import UnifiedState -from core.utils import measure_time, get_environment_variable +from app.core.logger import logger +from app.services.models import UnifiedState +from app.core.utils import measure_time, get_environment_variable +from app.core.logger import count_by_type_articles +from app.services.model_service import init_sentence_model THRESHOLD_SEMANTIC_SEARCH = float(get_environment_variable("THRESHOLD_SEMANTIC_SEARCH", "0.5")) @@ -25,7 +27,7 @@ def _filter_articles_with_faiss( :return: Articles filtrés """ import faiss - from services.model_service import init_sentence_model + model = init_sentence_model() logger.info(f"Filtrage sémantique avec les mots-clés {keywords}") @@ -93,7 +95,7 @@ def get_or_create_index(keywords, model, index_path): return filtered def filter_node(state: UnifiedState) -> UnifiedState: - from core.logger import count_by_type_articles + logger.info("🔍 Filtrage des articles par mots-clés...") filtered = _filter_articles_with_faiss( diff --git a/nodes/output_nodes.py b/app/nodes/output_nodes.py similarity index 94% rename from nodes/output_nodes.py rename to app/nodes/output_nodes.py index 8153f3e..209dca7 100644 --- a/nodes/output_nodes.py +++ b/app/nodes/output_nodes.py @@ -1,7 +1,7 @@ import logging logging.basicConfig(level=logging.INFO) from colorama import Fore -from core.logger import logger +from app.core.logger import logger from models.states import RSSState def output_node(state: RSSState) -> RSSState: diff --git a/nodes/save_nodes.py b/app/nodes/save_nodes.py similarity index 91% rename from nodes/save_nodes.py rename to app/nodes/save_nodes.py index 37da05e..4451869 100644 --- a/nodes/save_nodes.py +++ b/app/nodes/save_nodes.py @@ -1,7 +1,7 @@ import logging logging.basicConfig(level=logging.INFO) from colorama import Fore -from core.logger import logger +from app.core.logger import logger from models.states import RSSState def save_articles_node(state: RSSState) -> RSSState: diff --git a/nodes/send_nodes.py b/app/nodes/send_nodes.py similarity index 83% rename from nodes/send_nodes.py rename to app/nodes/send_nodes.py index 1b88a7b..51f11ca 100644 --- a/nodes/send_nodes.py +++ b/app/nodes/send_nodes.py @@ -1,18 +1,17 @@ -import os import logging logging.basicConfig(level=logging.INFO) from dotenv import load_dotenv -from core.logger import logger -from core.utils import get_environment_variable +from app.core.logger import logger +from app.core.utils import get_environment_variable from models.states import RSSState +from models.emails import EmailTemplateParams load_dotenv() THRESHOLD_SEMANTIC_SEARCH = float(get_environment_variable("THRESHOLD_SEMANTIC_SEARCH", "0.5")) def send_articles_node(state: RSSState) -> RSSState: from send_articles_email import send_watch_articles - from models.emails import EmailTemplateParams - + logger.info("Envoi mail des articles") logger.info(f"Envoi de {len(state.summaries)} articles") if len(state.summaries) > 0: diff --git a/nodes/summarize_nodes.py b/app/nodes/summarize_nodes.py similarity index 88% rename from nodes/summarize_nodes.py rename to app/nodes/summarize_nodes.py index 8bbb112..1405b5d 100644 --- a/nodes/summarize_nodes.py +++ b/app/nodes/summarize_nodes.py @@ -2,10 +2,14 @@ logging.basicConfig(level=logging.INFO) from colorama import Fore -from core.logger import logger -from services.models import UnifiedState -from core.utils import measure_time, get_environment_variable #, argscli -from services.model_service import set_prompt +from app.core.logger import logger +from app.services.models import UnifiedState +from app.core.utils import measure_time, get_environment_variable #, argscli +from app.services.model_service import set_prompt +from app.services.sources_ponderation import select_articles_for_summary +from app.core.logger import count_by_type_articles +from app.services.model_service import init_llm_chat +from app.core.utils import configure_logging_from_args def _calculate_tokens(summary, elapsed): """Calcule le nombre approximatif de tokens dans un texte et le débit en tokens/s.""" @@ -20,9 +24,7 @@ def _calculate_tokens(summary, elapsed): @measure_time def _summarize_article(title, content): import time - from services.model_service import init_llm_chat - from core.utils import configure_logging_from_args - + prompt = set_prompt("IA, ingénieurie logicielle et cybersécurité", title, content) _, args = configure_logging_from_args() if args.debug: @@ -65,8 +67,6 @@ def summarize_node(state: UnifiedState) -> UnifiedState: # dict Article : 'title', 'summary', 'link', 'published', 'score', 'source' # from datetime import datetime, timezone - from services.sources_ponderation import select_articles_for_summary - from core.logger import count_by_type_articles logger.info("✏️ Résumé des articles filtrés...") articles = state.filtered_articles diff --git a/nodes/utils_fetch_nodes.py b/app/nodes/utils_fetch_nodes.py similarity index 89% rename from nodes/utils_fetch_nodes.py rename to app/nodes/utils_fetch_nodes.py index dcd8be8..7cef07e 100644 --- a/nodes/utils_fetch_nodes.py +++ b/app/nodes/utils_fetch_nodes.py @@ -4,9 +4,9 @@ import logging logging.basicConfig(level=logging.INFO) -from core.logger import logger -from core.utils import get_environment_variable -from services.models import Source, SourceType, UnifiedState +from app.core.logger import logger +from app.core.utils import get_environment_variable +from app.services.models import Source, SourceType MAX_DAYS = int(get_environment_variable("MAX_DAYS", "10")) @@ -14,11 +14,11 @@ "get_rss_urls", "get_subs_reddit_urls", "get_bluesky_urls"] def register_fetchers(): - from services.factory_fetcher import FetcherFactory - from services.fetchers.reedit_fetcher import RedditFetcher - from services.fetchers.rss_fetcher import RSSFetcher - from services.fetchers.bluesky_fetcher import BlueskyFetcher - from services.models import SourceType + from app.services.factory_fetcher import FetcherFactory + from app.services.fetchers.reedit_fetcher import RedditFetcher + from app.services.fetchers.rss_fetcher import RSSFetcher + from app.services.fetchers.bluesky_fetcher import BlueskyFetcher + from app.services.models import SourceType FetcherFactory.register_fetcher(SourceType.RSS, RSSFetcher) FetcherFactory.register_fetcher(SourceType.REDDIT, RedditFetcher) @@ -30,7 +30,7 @@ def get_rss_urls(): Le .env ne contient que des types string et au format JSON """ from read_opml import parse_opml_to_rss_list - from services.models import Source, SourceType + from app.services.models import Source, SourceType OPML_FILE = get_environment_variable("OPML_FILE", "my.opml") logger.info("Obtention des URL RSS à traiter...") diff --git a/services/decorators.py b/app/services/decorators.py similarity index 95% rename from services/decorators.py rename to app/services/decorators.py index fd497cb..6bd29e9 100644 --- a/services/decorators.py +++ b/app/services/decorators.py @@ -2,7 +2,7 @@ import logging logging.basicConfig(level=logging.INFO) -from core.logger import logger +from app.core.logger import logger from colorama import Fore _FETCHERS = {} @@ -23,7 +23,7 @@ class RSSFetcher(BaseFetcher): env_flag = "RSS_FETCH" ... """ - from services.factory_fetcher import FetcherRegistry + from app.services.factory_fetcher import FetcherRegistry logger.info(Fore.YELLOW + f"🏷️ Décorateur @fetcher_class appelé pour: {cls.__name__}") FetcherRegistry.register(cls) logger.info(Fore.GREEN + f"✓ {cls.__name__} enregistré via décorateur") diff --git a/services/factory_fetcher.py b/app/services/factory_fetcher.py similarity index 94% rename from services/factory_fetcher.py rename to app/services/factory_fetcher.py index 7a08924..3b463f9 100644 --- a/services/factory_fetcher.py +++ b/app/services/factory_fetcher.py @@ -2,9 +2,9 @@ import logging logging.basicConfig(level=logging.INFO) -from core.logger import logger -from services.models import SourceType -from services.fetchers.base_fetcher import BaseFetcher +from app.core.logger import logger +from app.services.models import SourceType +from app.services.fetchers.base_fetcher import BaseFetcher class FetcherFactory: _fetchers = {} diff --git a/services/fetchers/__init__.py b/app/services/fetchers/__init__.py similarity index 100% rename from services/fetchers/__init__.py rename to app/services/fetchers/__init__.py diff --git a/services/fetchers/base_fetcher.py b/app/services/fetchers/base_fetcher.py similarity index 81% rename from services/fetchers/base_fetcher.py rename to app/services/fetchers/base_fetcher.py index d62af54..579b7c7 100644 --- a/services/fetchers/base_fetcher.py +++ b/app/services/fetchers/base_fetcher.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from services.models import Source +from app.services.models import Source class BaseFetcher(ABC): @abstractmethod diff --git a/services/fetchers/bluesky_fetcher.py b/app/services/fetchers/bluesky_fetcher.py similarity index 97% rename from services/fetchers/bluesky_fetcher.py rename to app/services/fetchers/bluesky_fetcher.py index 98c79e0..b805257 100644 --- a/services/fetchers/bluesky_fetcher.py +++ b/app/services/fetchers/bluesky_fetcher.py @@ -5,12 +5,13 @@ import logging logging.basicConfig(level=logging.INFO) -from core.logger import logger, Fore -from core import measure_time +from app.core.logger import logger, Fore +from app.core import measure_time -from services.decorators import fetcher_class -from services.fetchers.base_fetcher import BaseFetcher -from services.models import Source, SourceType +from app.services.decorators import fetcher_class +from app.services.fetchers.base_fetcher import BaseFetcher +from app.services.models import Source, SourceType +from app.core.logger import print_color # class ArticleBluesky(BaseModel): # title: str @@ -63,9 +64,7 @@ def fetch_articles(self, source: Source, max_days: int) -> list[dict]: - Un handle utilisateur : "@user.bsky.social" - Un DID : "did:plc:..." - "firehose" pour le feed public général - """ - from core.logger import print_color - + """ color = Fore.LIGHTMAGENTA_EX print_color(color, "=" * 60) print_color(color, f"BLUESKY Fetcher fetch_articles {source.url}") diff --git a/services/fetchers/reedit_fetcher.py b/app/services/fetchers/reedit_fetcher.py similarity index 90% rename from services/fetchers/reedit_fetcher.py rename to app/services/fetchers/reedit_fetcher.py index 32c5f27..3b089fc 100644 --- a/services/fetchers/reedit_fetcher.py +++ b/app/services/fetchers/reedit_fetcher.py @@ -4,13 +4,14 @@ import logging logging.basicConfig(level=logging.INFO) -from services.decorators import fetcher_class -from services.fetchers.base_fetcher import BaseFetcher -from services.models import Source, SourceType +from app.services.decorators import fetcher_class +from app.services.fetchers.base_fetcher import BaseFetcher +from app.services.models import Source, SourceType -from core.logger import logger, Fore -from core.utils import get_environment_variable -from core import measure_time +from app.core.logger import logger, Fore +from app.core.utils import get_environment_variable +from app.core import measure_time +from app.core.logger import print_color @fetcher_class class RedditFetcher(BaseFetcher): @@ -26,8 +27,7 @@ def __init__(self, client_id: str, client_secret: str, user_agent: str): self.max_fetch = int(get_environment_variable("REDDIT_MAX_FETCH", 10)) @measure_time - def fetch_articles(self, source: Source, max_days: int) -> list[dict]: - from core.logger import print_color + def fetch_articles(self, source: Source, max_days: int) -> list[dict]: articles = [] cutoff_date = datetime.now() - timedelta(days=max_days) subreddit = self.reddit.subreddit(source.subreddit) diff --git a/services/fetchers/rss_fetcher.py b/app/services/fetchers/rss_fetcher.py similarity index 93% rename from services/fetchers/rss_fetcher.py rename to app/services/fetchers/rss_fetcher.py index 1977e9a..0e8e1e6 100644 --- a/services/fetchers/rss_fetcher.py +++ b/app/services/fetchers/rss_fetcher.py @@ -3,12 +3,13 @@ import logging logging.basicConfig(level=logging.INFO) -from core.logger import logger, Fore -from core import measure_time +from app.core.logger import logger, Fore +from app.core import measure_time -from services.decorators import fetcher_class -from services.fetchers.base_fetcher import BaseFetcher -from services.models import Source, SourceType +from app.services.decorators import fetcher_class +from app.services.fetchers.base_fetcher import BaseFetcher +from app.services.models import Source, SourceType +from app.core.logger import print_color @fetcher_class class RSSFetcher(BaseFetcher): @@ -86,8 +87,7 @@ def add_article_with_entry_syndication(self, entry, articles, cutoff_date, recen @measure_time def fetch_articles(self, source: Source, max_days: int) -> list[dict]: - """Votre logique RSS existante""" - from core.logger import print_color + """Votre logique RSS existante""" AGENT = "ReaderRSS/1.0" RESOLVE_RELATIVE_URIS = False SANITIZE_HTML = True diff --git a/services/model_service.py b/app/services/model_service.py similarity index 98% rename from services/model_service.py rename to app/services/model_service.py index 923f42a..701f38d 100644 --- a/services/model_service.py +++ b/app/services/model_service.py @@ -7,8 +7,8 @@ from colorama import Fore -from core.logger import logger -from core.utils import get_environment_variable +from app.core.logger import logger +from app.core.utils import get_environment_variable # ========================= # Configuration du modèle LLM inférence diff --git a/services/models.py b/app/services/models.py similarity index 100% rename from services/models.py rename to app/services/models.py diff --git a/services/sources_ponderation.py b/app/services/sources_ponderation.py similarity index 96% rename from services/sources_ponderation.py rename to app/services/sources_ponderation.py index e65ed15..4e97e3e 100644 --- a/services/sources_ponderation.py +++ b/app/services/sources_ponderation.py @@ -1,11 +1,13 @@ from math import ceil import logging -from services.models import SourceType +from app.services.models import SourceType logging.basicConfig(level=logging.INFO) -from core.logger import logger, Fore -from core import get_environment_variable +from app.core.logger import logger, Fore +from app.core import get_environment_variable +from app.core.logger import print_color +from app.core.logger import count_by_type_articles def _calculate_quotas(total_count): """Calcul des quotas par source selon % donnés en .env""" @@ -64,7 +66,7 @@ def _fill_flexible_slots(remaining_articles, flexible_slots, processed_sources: def _count_by_type_articles(title, articles): from collections import Counter - from core.logger import print_color + source_counts = Counter(item['source'].value for item in articles) color = Fore.LIGHTYELLOW_EX print_color(color, "=" * 60) @@ -81,9 +83,7 @@ def select_articles_for_summary(articles_by_source: list[dict]) -> list[dict]: Returns: Liste des articles sélectionnés et équilibrés - """ - from core.logger import count_by_type_articles - + """ count_by_type_articles(f"Nombre d'articles sources {len(articles_by_source)}", articles_by_source) limit_articles_to_resume = int(get_environment_variable('LIMIT_ARTICLES_TO_RESUME', 15)) diff --git a/services/utils_fetchers.py b/app/services/utils_fetchers.py similarity index 91% rename from services/utils_fetchers.py rename to app/services/utils_fetchers.py index a95c24b..47bac67 100644 --- a/services/utils_fetchers.py +++ b/app/services/utils_fetchers.py @@ -1,15 +1,17 @@ import logging logging.basicConfig(level=logging.INFO) -from core.logger import logger from colorama import Fore +from app.core.logger import logger +from app.services.factory_fetcher import FetcherRegistry + def register_fetchers_auto(): """Découvre automatiquement tous les BaseFetcher du module.""" import importlib import pkgutil - from services.factory_fetcher import FetcherRegistry + - MODULE_FETCHERS = "services.fetchers" + MODULE_FETCHERS = "app.services.fetchers" logger.info(Fore.CYAN + f"🔍 Découverte automatique dans {MODULE_FETCHERS}") fetchers_package = importlib.import_module(MODULE_FETCHERS) diff --git a/db/db.py b/db/db.py index 1fb8ffc..b792adf 100644 --- a/db/db.py +++ b/db/db.py @@ -11,7 +11,7 @@ from dotenv import load_dotenv from datetime import datetime, timezone from models.article import ArticleModel -from services.models import SourceType +from app.services.models import SourceType load_dotenv() DB_PATH = os.getenv("DB_PATH", "techno-watch.db") diff --git a/jinja_filters.py b/jinja_filters.py index 8f59e4a..1736e3c 100644 --- a/jinja_filters.py +++ b/jinja_filters.py @@ -6,7 +6,7 @@ import base64 import logging -from services.models import SourceType +from app.services.models import SourceType logging.basicConfig(level=logging.INFO) diff --git a/models/article.py b/models/article.py index 825ca8f..adc61ff 100644 --- a/models/article.py +++ b/models/article.py @@ -1,6 +1,6 @@ from pydantic import BaseModel -from services.models import SourceType +from app.services.models import SourceType class ArticleModel(BaseModel): """Modèle Pydantic de vérification""" diff --git a/read_opml.py b/read_opml.py index b92342b..82e8f4f 100644 --- a/read_opml.py +++ b/read_opml.py @@ -3,7 +3,7 @@ import xml.etree.ElementTree as ET logging.basicConfig(level=logging.INFO) -from core.logger import logger +from app.core.logger import logger class RSSFeed(NamedTuple): titre: str diff --git a/send_articles_email.py b/send_articles_email.py index 8b4dce0..d9f9e3f 100644 --- a/send_articles_email.py +++ b/send_articles_email.py @@ -6,7 +6,7 @@ from jinja_filters import register_jinja_filters from models.emails import EmailTemplateParams -from core.utils import get_environment_variable +from app.core.utils import get_environment_variable # ========================= # Configuration SMTP diff --git a/tests/conftest.py b/tests/conftest.py index 33591fd..ca69999 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,6 @@ import pytest import sys -from services.models import SourceType, UnifiedState +from app.services.models import SourceType # @pytest.fixture # def base_state(): diff --git a/tests/test_sources_ponderation.py b/tests/test_sources_ponderation.py index e8f7f35..4ebc62a 100644 --- a/tests/test_sources_ponderation.py +++ b/tests/test_sources_ponderation.py @@ -1,7 +1,7 @@ from unittest.mock import patch -from nodes.summarize_nodes import summarize_node -from services.models import SourceType, UnifiedState -from services.sources_ponderation import select_articles_for_summary +from app.nodes.summarize_nodes import summarize_node +from app.services.models import SourceType, UnifiedState +from app.services.sources_ponderation import select_articles_for_summary def test_select_articles_for_summary_basic(mock_articles, mock_env_vars): @@ -39,8 +39,9 @@ def test_select_articles_for_summary_limit(mock_articles, mock_env_vars): def test_summarize_node_limit(mock_articles, mock_env_vars): # state = UnifiedState(filtered_articles=mock_articles) + PATCH="app.nodes.summarize_nodes._summarize_article" state = UnifiedState(filtered_articles=mock_articles, keywords=[]) - with patch("nodes.summarize_nodes._summarize_article", return_value="Mock summary"): + with patch(PATCH, return_value="Mock summary"): result = summarize_node(state) assert len(result.summaries) == 5 assert all(s["summary"] == "Mock summary" for s in result.summaries) \ No newline at end of file diff --git a/web.py b/web.py index 3930a97..5b8a57e 100644 --- a/web.py +++ b/web.py @@ -14,7 +14,7 @@ import logging -from services.models import SourceType +from app.services.models import SourceType logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) From ce5593bc04f722a56e63747e65f066ed3d614d35 Mon Sep 17 00:00:00 2001 From: Olivier DUVAL Date: Sat, 25 Oct 2025 18:00:57 +0200 Subject: [PATCH 03/13] models -> app db -> app --- app/__init__.py | 4 ++-- {db => app/db}/__init__.py | 4 ++-- {db => app/db}/db.py | 3 ++- app/main_agent.py | 4 +--- {models => app/models}/article.py | 0 {models => app/models}/emails.py | 0 {models => app/models}/states.py | 0 app/nodes/output_nodes.py | 2 +- app/nodes/save_nodes.py | 2 +- app/nodes/send_nodes.py | 4 ++-- send_articles_email.py | 4 ++-- web.py | 6 ++++-- 12 files changed, 17 insertions(+), 16 deletions(-) rename {db => app/db}/__init__.py (50%) rename {db => app/db}/db.py (99%) rename {models => app/models}/article.py (100%) rename {models => app/models}/emails.py (100%) rename {models => app/models}/states.py (100%) diff --git a/app/__init__.py b/app/__init__.py index 6202c0d..e9e8a4d 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -1,8 +1,8 @@ # from core import measure_time, argscli, logger from .core.logger import logger from .core.utils import measure_time #, get_environment_variable -from models.emails import EmailTemplateParams -from models.states import RSSState +from .models.emails import EmailTemplateParams +from .models.states import RSSState __all__ = [ 'logger', diff --git a/db/__init__.py b/app/db/__init__.py similarity index 50% rename from db/__init__.py rename to app/db/__init__.py index a72f598..6bcedc5 100644 --- a/db/__init__.py +++ b/app/db/__init__.py @@ -1,5 +1,5 @@ -from db.db import init_db, save_to_db, DB_PATH -from db.db import read_articles +from .db import init_db, save_to_db, DB_PATH +from .db import read_articles __all__=[ 'init_db', diff --git a/db/db.py b/app/db/db.py similarity index 99% rename from db/db.py rename to app/db/db.py index b792adf..21b94e1 100644 --- a/db/db.py +++ b/app/db/db.py @@ -10,7 +10,8 @@ from contextlib import contextmanager from dotenv import load_dotenv from datetime import datetime, timezone -from models.article import ArticleModel + +from app.models.article import ArticleModel from app.services.models import SourceType load_dotenv() diff --git a/app/main_agent.py b/app/main_agent.py index 9dc4ace..ad60297 100644 --- a/app/main_agent.py +++ b/app/main_agent.py @@ -27,10 +27,8 @@ # from langchain_openai import ChatOpenAI # from langchain_ollama import ChatOllama -from models.states import RSSState -# from read_opml import parse_opml_to_rss_list +from .models.states import RSSState -# from core import argscli from .services.utils_fetchers import register_fetchers_auto from .services.models import SourceType, UnifiedState diff --git a/models/article.py b/app/models/article.py similarity index 100% rename from models/article.py rename to app/models/article.py diff --git a/models/emails.py b/app/models/emails.py similarity index 100% rename from models/emails.py rename to app/models/emails.py diff --git a/models/states.py b/app/models/states.py similarity index 100% rename from models/states.py rename to app/models/states.py diff --git a/app/nodes/output_nodes.py b/app/nodes/output_nodes.py index 209dca7..ada7741 100644 --- a/app/nodes/output_nodes.py +++ b/app/nodes/output_nodes.py @@ -2,7 +2,7 @@ logging.basicConfig(level=logging.INFO) from colorama import Fore from app.core.logger import logger -from models.states import RSSState +from app.models.states import RSSState def output_node(state: RSSState) -> RSSState: logger.info("📄 Affichage des résultats finaux") diff --git a/app/nodes/save_nodes.py b/app/nodes/save_nodes.py index 4451869..d9e10c4 100644 --- a/app/nodes/save_nodes.py +++ b/app/nodes/save_nodes.py @@ -2,7 +2,7 @@ logging.basicConfig(level=logging.INFO) from colorama import Fore from app.core.logger import logger -from models.states import RSSState +from app.models.states import RSSState def save_articles_node(state: RSSState) -> RSSState: from db.db import save_to_db diff --git a/app/nodes/send_nodes.py b/app/nodes/send_nodes.py index 51f11ca..d2d4c01 100644 --- a/app/nodes/send_nodes.py +++ b/app/nodes/send_nodes.py @@ -3,8 +3,8 @@ from dotenv import load_dotenv from app.core.logger import logger from app.core.utils import get_environment_variable -from models.states import RSSState -from models.emails import EmailTemplateParams +from app.models.states import RSSState +from app.models.emails import EmailTemplateParams load_dotenv() THRESHOLD_SEMANTIC_SEARCH = float(get_environment_variable("THRESHOLD_SEMANTIC_SEARCH", "0.5")) diff --git a/send_articles_email.py b/send_articles_email.py index d9f9e3f..c5b5325 100644 --- a/send_articles_email.py +++ b/send_articles_email.py @@ -5,7 +5,7 @@ from email.mime.text import MIMEText from jinja_filters import register_jinja_filters -from models.emails import EmailTemplateParams +from app.models.emails import EmailTemplateParams from app.core.utils import get_environment_variable # ========================= @@ -91,7 +91,7 @@ def send_watch_articles(params: EmailTemplateParams): ) if __name__ == "__main__": - from models.emails import EmailTemplateParams + from app. import EmailTemplateParams articles = [ {'title': "L'IA révolutionne la médecine en 2025", 'link': 'https://domain.ntld', 'summary': 'résumé', 'score': '60 %'}, {'title': "Comment l'énergie solaire transforme les villes", 'link': 'https://domain2.ntld', 'summary': 'résumé', 'score': '45 %'}, diff --git a/web.py b/web.py index 5b8a57e..d4d14dd 100644 --- a/web.py +++ b/web.py @@ -15,6 +15,8 @@ import logging from app.services.models import SourceType +from app.db.db import ArticleFTS, get_db + logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -30,7 +32,7 @@ @app.get("/") def read_articles(request: Request, date: str = None): """Affiche les articles filtrés par date de publication.""" - from db import read_articles + from app.db import read_articles articles = read_articles(date) logger.debug(f"Articles lus: len({articles})") # for article in articles: @@ -56,7 +58,7 @@ def search_articles( date_min/date_max: Filtres de date (format YYYY-MM-DD) limit: Nombre max de résultats """ - from db.db import ArticleFTS, get_db + from datetime import datetime if not q: From 44f7047d8abedd5b2ac32b93b7031ae7c1ce9aaf Mon Sep 17 00:00:00 2001 From: Olivier DUVAL Date: Sun, 26 Oct 2025 13:53:41 +0100 Subject: [PATCH 04/13] jinja_filters.py, send_articles_email.py => app templates => app tests/test_send_articles_email.py --- jinja_filters.py => app/jinja_filters.py | 5 +- app/main_agent.py | 4 +- app/models/__init__.py | 3 + app/nodes/send_nodes.py | 2 +- .../send_articles_email.py | 20 +- .../templates}/email_template.html.j2 | 0 .../templates}/email_template.text.j2 | 0 .../web/fragments/_dates_filters.html | 0 .../templates}/web/fragments/_icon_brand.html | 0 .../web/fragments/_search_ajax_results.html | 0 .../web/fragments/_search_banner.html | 0 .../templates}/web/fragments/_toc.html | 0 .../templates}/web/icons/bluesky.svg | 0 .../templates}/web/icons/linkedin.svg | 0 .../templates}/web/icons/reddit.svg | 0 .../templates}/web/icons/rss.svg | 0 .../templates}/web/icons/twitter.svg | 0 {templates => app/templates}/web/index.html | 0 {templates => app/templates}/web/styles.css | 0 tests/conftest.py | 1 + tests/test_send_articles_email.py | 390 ++++++++++++++++++ web.py | 8 +- 22 files changed, 418 insertions(+), 15 deletions(-) rename jinja_filters.py => app/jinja_filters.py (97%) create mode 100644 app/models/__init__.py rename send_articles_email.py => app/send_articles_email.py (86%) rename {templates => app/templates}/email_template.html.j2 (100%) rename {templates => app/templates}/email_template.text.j2 (100%) rename {templates => app/templates}/web/fragments/_dates_filters.html (100%) rename {templates => app/templates}/web/fragments/_icon_brand.html (100%) rename {templates => app/templates}/web/fragments/_search_ajax_results.html (100%) rename {templates => app/templates}/web/fragments/_search_banner.html (100%) rename {templates => app/templates}/web/fragments/_toc.html (100%) rename {templates => app/templates}/web/icons/bluesky.svg (100%) rename {templates => app/templates}/web/icons/linkedin.svg (100%) rename {templates => app/templates}/web/icons/reddit.svg (100%) rename {templates => app/templates}/web/icons/rss.svg (100%) rename {templates => app/templates}/web/icons/twitter.svg (100%) rename {templates => app/templates}/web/index.html (100%) rename {templates => app/templates}/web/styles.css (100%) create mode 100644 tests/test_send_articles_email.py diff --git a/jinja_filters.py b/app/jinja_filters.py similarity index 97% rename from jinja_filters.py rename to app/jinja_filters.py index 1736e3c..272090e 100644 --- a/jinja_filters.py +++ b/app/jinja_filters.py @@ -39,7 +39,7 @@ def nl2br(value): @lru_cache(maxsize=32) def get_svg_base64(source: str) -> str: - """cache des icones SVG en base64 pour les emails.""" + """cache des icones SVG en base64 pour les emails.""" icon_path = Path(__file__).parent / "templates" / "web" / "icons" / f"{source}.svg" with open(icon_path, "rb") as f: return base64.b64encode(f.read()).decode("utf-8") @@ -52,8 +52,7 @@ def icon_html(source: SourceType, size: int = 24, email: bool = False) -> str: size: Taille en pixels (défaut: 24). email: Si True, retourne une balise avec SVG en base64. """ - logging.debug(f"icon_html: source={source}, size={size}, email={email}") - + logging.debug(f"icon_html: source={source}, size={size}, email={email}") icon_path = Path(__file__).parent / "templates" / "web" / "icons" / f"{source}.svg" FALLBACK_EMOJIS = { "rss": "📡", diff --git a/app/main_agent.py b/app/main_agent.py index ad60297..6476a62 100644 --- a/app/main_agent.py +++ b/app/main_agent.py @@ -277,7 +277,7 @@ def prepare_data(): # Main # ========================= def main(): - from db.db import init_db + from app.db import init_db _, args = configure_logging_from_args() logger.info(Fore.MAGENTA + Style.BRIGHT + "=== Agent RSS avec résumés LLM ===") @@ -302,7 +302,7 @@ def main(): def search(): - from db.db import ( + from app.db import ( search_fts, init_db, ) diff --git a/app/models/__init__.py b/app/models/__init__.py new file mode 100644 index 0000000..71331d5 --- /dev/null +++ b/app/models/__init__.py @@ -0,0 +1,3 @@ +from app.models.emails import EmailTemplateParams + +__all__ = ['EmailTemplateParams'] \ No newline at end of file diff --git a/app/nodes/send_nodes.py b/app/nodes/send_nodes.py index d2d4c01..f5a9c45 100644 --- a/app/nodes/send_nodes.py +++ b/app/nodes/send_nodes.py @@ -10,7 +10,7 @@ THRESHOLD_SEMANTIC_SEARCH = float(get_environment_variable("THRESHOLD_SEMANTIC_SEARCH", "0.5")) def send_articles_node(state: RSSState) -> RSSState: - from send_articles_email import send_watch_articles + from app.send_articles_email import send_watch_articles logger.info("Envoi mail des articles") logger.info(f"Envoi de {len(state.summaries)} articles") diff --git a/send_articles_email.py b/app/send_articles_email.py similarity index 86% rename from send_articles_email.py rename to app/send_articles_email.py index c5b5325..e9fa459 100644 --- a/send_articles_email.py +++ b/app/send_articles_email.py @@ -1,11 +1,15 @@ +from pathlib import Path +from colorama import Fore from jinja2 import Environment, FileSystemLoader from datetime import datetime import smtplib from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText -from jinja_filters import register_jinja_filters -from app.models.emails import EmailTemplateParams +from app.core.logger import print_color +from app.jinja_filters import register_jinja_filters +from app.models import EmailTemplateParams +# from app.models.emails import EmailTemplateParams from app.core.utils import get_environment_variable # ========================= @@ -23,9 +27,13 @@ # Rendu du template Jinja2 # ========================= +PATH = Path(__file__).parent / "templates" + def _set_env_render_filters(): + print_color(Fore.GREEN, f"Setting Jinja2 template path to: {PATH}") + loader = FileSystemLoader(PATH) env = Environment( - loader=FileSystemLoader("templates"), + loader=loader, autoescape=True ) register_jinja_filters(env) @@ -91,10 +99,10 @@ def send_watch_articles(params: EmailTemplateParams): ) if __name__ == "__main__": - from app. import EmailTemplateParams + """Test d'envoi d'email avec des articles fictifs.""" articles = [ - {'title': "L'IA révolutionne la médecine en 2025", 'link': 'https://domain.ntld', 'summary': 'résumé', 'score': '60 %'}, - {'title': "Comment l'énergie solaire transforme les villes", 'link': 'https://domain2.ntld', 'summary': 'résumé', 'score': '45 %'}, + {'title': "L'IA révolutionne la médecine en 2025", 'link': 'https://domain.ntld', 'summary': 'résumé', 'score': '60', 'published': '2025-01-01T20:30:00', 'source': 'rss'}, + {'title': "Comment l'énergie solaire transforme les villes", 'link': 'https://domain2.ntld', 'summary': 'résumé', 'score': '45', 'published': '2025-01-02T10:15:00', 'source': 'rss'}, # "Les dernières avancées en robotique industrielle", # "Comment l'énergie solaire transforme les villes", # "La cybersécurité face aux nouvelles menaces", diff --git a/templates/email_template.html.j2 b/app/templates/email_template.html.j2 similarity index 100% rename from templates/email_template.html.j2 rename to app/templates/email_template.html.j2 diff --git a/templates/email_template.text.j2 b/app/templates/email_template.text.j2 similarity index 100% rename from templates/email_template.text.j2 rename to app/templates/email_template.text.j2 diff --git a/templates/web/fragments/_dates_filters.html b/app/templates/web/fragments/_dates_filters.html similarity index 100% rename from templates/web/fragments/_dates_filters.html rename to app/templates/web/fragments/_dates_filters.html diff --git a/templates/web/fragments/_icon_brand.html b/app/templates/web/fragments/_icon_brand.html similarity index 100% rename from templates/web/fragments/_icon_brand.html rename to app/templates/web/fragments/_icon_brand.html diff --git a/templates/web/fragments/_search_ajax_results.html b/app/templates/web/fragments/_search_ajax_results.html similarity index 100% rename from templates/web/fragments/_search_ajax_results.html rename to app/templates/web/fragments/_search_ajax_results.html diff --git a/templates/web/fragments/_search_banner.html b/app/templates/web/fragments/_search_banner.html similarity index 100% rename from templates/web/fragments/_search_banner.html rename to app/templates/web/fragments/_search_banner.html diff --git a/templates/web/fragments/_toc.html b/app/templates/web/fragments/_toc.html similarity index 100% rename from templates/web/fragments/_toc.html rename to app/templates/web/fragments/_toc.html diff --git a/templates/web/icons/bluesky.svg b/app/templates/web/icons/bluesky.svg similarity index 100% rename from templates/web/icons/bluesky.svg rename to app/templates/web/icons/bluesky.svg diff --git a/templates/web/icons/linkedin.svg b/app/templates/web/icons/linkedin.svg similarity index 100% rename from templates/web/icons/linkedin.svg rename to app/templates/web/icons/linkedin.svg diff --git a/templates/web/icons/reddit.svg b/app/templates/web/icons/reddit.svg similarity index 100% rename from templates/web/icons/reddit.svg rename to app/templates/web/icons/reddit.svg diff --git a/templates/web/icons/rss.svg b/app/templates/web/icons/rss.svg similarity index 100% rename from templates/web/icons/rss.svg rename to app/templates/web/icons/rss.svg diff --git a/templates/web/icons/twitter.svg b/app/templates/web/icons/twitter.svg similarity index 100% rename from templates/web/icons/twitter.svg rename to app/templates/web/icons/twitter.svg diff --git a/templates/web/index.html b/app/templates/web/index.html similarity index 100% rename from templates/web/index.html rename to app/templates/web/index.html diff --git a/templates/web/styles.css b/app/templates/web/styles.css similarity index 100% rename from templates/web/styles.css rename to app/templates/web/styles.css diff --git a/tests/conftest.py b/tests/conftest.py index ca69999..676f7b6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -26,3 +26,4 @@ def mock_env_vars(monkeypatch): monkeypatch.setenv("BLUESKY_WEIGHT", "20") monkeypatch.setenv("LIMIT_ARTICLES_TO_RESUME", "5") monkeypatch.setenv("FRESHNESS_BOOST_THRESHOLD", "0.3") + diff --git a/tests/test_send_articles_email.py b/tests/test_send_articles_email.py new file mode 100644 index 0000000..d153ee3 --- /dev/null +++ b/tests/test_send_articles_email.py @@ -0,0 +1,390 @@ +"""Tests d'intégration pour l'envoi d'emails de veille technologique.""" +""" +# Tous les tests +pytest tests/test_send_articles_email.py -v + +# Tests d'une classe spécifique +pytest tests/test_send_articles_email.py::TestEmailTemplateRendering -v + +# Avec couverture +pytest tests/test_send_articles_email.py --cov=app.send_articles_email + +# Test manuel (avec vraie config SMTP) +pytest tests/test_send_articles_email.py -m manual --no-skip +""" +import pytest +from unittest.mock import Mock, patch, MagicMock +from datetime import datetime +import smtplib + +from app.send_articles_email import ( + render_email_template, + send_watch_articles, + _send_email, +) +from app.models.emails import EmailTemplateParams + + +# ========================= +# Fixtures +# ========================= + +@pytest.fixture(autouse=True) +def mock_icon_filter(monkeypatch): + """Mock le filtre d'icônes pour éviter les erreurs de fichiers manquants.""" + def mock_load_icon(icon_name): + return f'' + + # Si vous avez accès au module jinja_filters + from app import jinja_filters + if hasattr(jinja_filters, 'load_icon'): + monkeypatch.setattr(jinja_filters, 'load_icon', mock_load_icon) + + +@pytest.fixture +def sample_articles(): + """Articles de test.""" + return [ + { + 'title': "L'IA révolutionne la médecine en 2025", + 'link': 'https://domain.ntld', + 'summary': 'Un résumé sur les avancées de l\'IA en médecine', + 'score': '60', + 'published': '2025-01-01T20:30:00', + 'source': 'rss' + }, + { + 'title': "Comment l'énergie solaire transforme les villes", + 'link': 'https://domain2.ntld', + 'summary': 'Les innovations dans l\'énergie solaire urbaine', + 'score': '45', + 'published': '2025-01-02T10:15:00', + 'source': 'rss' + }, + ] + + +@pytest.fixture +def email_params(sample_articles): + """Paramètres d'email de test.""" + return EmailTemplateParams( + articles=sample_articles, + keywords=['ia', 'agent', 'énergie'], + threshold=0.5 + ) + + +# ========================= +# Tests unitaires du rendu +# ========================= + +class TestEmailTemplateRendering: + """Tests du rendu des templates Jinja2.""" + + def test_render_html_template(self, email_params): + """Teste le rendu du template HTML.""" + html_content = render_email_template(email_params, "email_template.html.j2") + + assert html_content is not None + assert len(html_content) > 0 + # Les apostrophes sont échappées en HTML (L'IA devient L'IA) + assert ("L'IA révolutionne la médecine en 2025" in html_content + or "L'IA révolutionne la médecine en 2025" in html_content) + assert "https://domain.ntld" in html_content + assert "50%" in html_content # threshold formaté + assert "ia" in html_content.lower() + assert "agent" in html_content.lower() + + def test_render_text_template(self, email_params): + """Teste le rendu du template texte.""" + text_content = render_email_template(email_params, "email_template.text.j2") + + assert text_content is not None + assert len(text_content) > 0 + # Dans le template texte, les apostrophes peuvent être échappées ou non selon le template + assert ("L'IA révolutionne" in text_content + or "L'IA révolutionne" in text_content + or "IA révolutionne" in text_content) + assert "https://domain.ntld" in text_content + assert "50%" in text_content + + def test_render_template_with_date(self, email_params): + """Vérifie que la date est bien incluse dans le rendu.""" + html_content = render_email_template(email_params, "email_template.html.j2") + + # La date doit être au format dd/mm/yyyy + current_date = datetime.now().strftime("%d/%m/%Y") + assert current_date in html_content + + def test_render_template_with_empty_articles(self): + """Teste le rendu avec une liste d'articles vide.""" + empty_params = EmailTemplateParams( + articles=[], + keywords=['test'], + threshold=0.3 + ) + + html_content = render_email_template(empty_params, "email_template.html.j2") + assert html_content is not None + # Le template devrait gérer gracieusement la liste vide + + +# ========================= +# Tests de l'envoi d'email +# ========================= + +class TestEmailSending: + """Tests de l'envoi d'emails.""" + + @patch('app.send_articles_email.smtplib.SMTP') + def test_send_email_success(self, mock_smtp): + """Teste l'envoi réussi d'un email.""" + # Mock du serveur SMTP + mock_server = MagicMock() + mock_smtp.return_value.__enter__.return_value = mock_server + + # Appel de la fonction + _send_email( + subject="Test Subject", + html_content="Test HTML", + text_content="Test Text", + sender="test@sender.com", + to="test@recipient.com", + smtp_server="smtp.test.com", + smtp_port=587, + login="testuser", + password="testpass", + ) + + # Vérifications + mock_smtp.assert_called_once_with("smtp.test.com", 587) + mock_server.ehlo.assert_called() + mock_server.starttls.assert_called_once() + mock_server.login.assert_called_once_with("testuser", "testpass") + mock_server.sendmail.assert_called_once() + + # Vérifier les arguments de sendmail + call_args = mock_server.sendmail.call_args[0] + assert call_args[0] == "test@sender.com" + assert call_args[1] == "test@recipient.com" + assert "Test Subject" in call_args[2] + + @patch('app.send_articles_email.smtplib.SMTP') + def test_send_email_contains_both_html_and_text(self, mock_smtp): + """Vérifie que l'email contient bien les parties HTML et texte.""" + mock_server = MagicMock() + mock_smtp.return_value.__enter__.return_value = mock_server + + _send_email( + subject="Test", + html_content="

HTML Content

", + text_content="Text Content", + sender="sender@test.com", + to="recipient@test.com", + smtp_server="smtp.test.com", + smtp_port=587, + login="user", + password="pass", + ) + + # Récupérer le message envoyé + sent_message = mock_server.sendmail.call_args[0][2] + assert "text/plain" in sent_message + assert "text/html" in sent_message + assert "HTML Content" in sent_message + assert "Text Content" in sent_message + + @patch('app.send_articles_email.smtplib.SMTP') + def test_send_email_smtp_connection_error(self, mock_smtp): + """Teste la gestion d'erreur de connexion SMTP.""" + mock_smtp.side_effect = smtplib.SMTPConnectError(421, "Cannot connect") + + with pytest.raises(smtplib.SMTPConnectError): + _send_email( + subject="Test", + html_content="

Test

", + text_content="Test", + sender="sender@test.com", + to="recipient@test.com", + smtp_server="invalid.smtp.com", + smtp_port=587, + login="user", + password="pass", + ) + + @patch('app.send_articles_email.smtplib.SMTP') + def test_send_email_authentication_error(self, mock_smtp): + """Teste la gestion d'erreur d'authentification.""" + mock_server = MagicMock() + mock_server.login.side_effect = smtplib.SMTPAuthenticationError(535, "Invalid credentials") + mock_smtp.return_value.__enter__.return_value = mock_server + + with pytest.raises(smtplib.SMTPAuthenticationError): + _send_email( + subject="Test", + html_content="

Test

", + text_content="Test", + sender="sender@test.com", + to="recipient@test.com", + smtp_server="smtp.test.com", + smtp_port=587, + login="baduser", + password="badpass", + ) + + +# ========================= +# Tests d'intégration +# ========================= + +class TestSendWatchArticlesIntegration: + """Tests d'intégration de bout en bout.""" + + @patch('app.send_articles_email.smtplib.SMTP') + @patch('app.send_articles_email.render_email_template') + def test_send_watch_articles_full_flow(self, mock_render, mock_smtp, email_params): + """Teste le flux complet d'envoi d'articles de veille.""" + # Mock des rendus de template + mock_render.side_effect = [ + "Rendered HTML", # Premier appel (HTML) + "Rendered Text Content" # Deuxième appel (Text) + ] + + # Mock du serveur SMTP + mock_server = MagicMock() + mock_smtp.return_value.__enter__.return_value = mock_server + + # Appel de la fonction principale + send_watch_articles(email_params) + + # Vérifications du rendu + assert mock_render.call_count == 2 + mock_render.assert_any_call(email_params, "email_template.html.j2") + mock_render.assert_any_call(email_params, "email_template.text.j2") + + # Vérifications de l'envoi SMTP + mock_server.login.assert_called_once() + mock_server.sendmail.assert_called_once() + + @patch('app.send_articles_email.smtplib.SMTP') + def test_send_watch_articles_with_real_templates(self, mock_smtp, email_params): + """Teste avec le rendu réel des templates (sans mock du rendu).""" + mock_server = MagicMock() + mock_smtp.return_value.__enter__.return_value = mock_server + + # Appel sans mocker le rendu + send_watch_articles(email_params) + + # Vérifier que l'email a été envoyé + mock_server.sendmail.assert_called_once() + + # Récupérer le contenu de l'email + sent_message = mock_server.sendmail.call_args[0][2] + + # Vérifier que le contenu contient nos articles + assert "L'IA révolutionne la médecine" in sent_message + assert "énergie solaire" in sent_message + assert "[VEILLE]" in sent_message + + @patch('app.send_articles_email.smtplib.SMTP') + def test_send_watch_articles_subject_format(self, mock_smtp, email_params): + """Vérifie le format du sujet de l'email.""" + mock_server = MagicMock() + mock_smtp.return_value.__enter__.return_value = mock_server + + send_watch_articles(email_params) + + sent_message = mock_server.sendmail.call_args[0][2] + + # Vérifier le format du sujet + assert "Subject: [VEILLE] Revue de veille techno du" in sent_message + # La date devrait être au format dd/mm/yyyy HH:MM:SS + current_date = datetime.now().strftime("%d/%m/%Y") + assert current_date in sent_message + + +# ========================= +# Tests de validation des données +# ========================= + +class TestEmailDataValidation: + """Tests de validation des données d'entrée.""" + + def test_email_params_with_missing_article_fields(self): + """Teste le comportement avec des champs manquants dans les articles.""" + incomplete_articles = [ + {'title': 'Article sans lien', 'summary': 'Test'} + # Manque: link, score, published, source + ] + + params = EmailTemplateParams( + articles=incomplete_articles, + keywords=['test'], + threshold=0.5 + ) + + # Le rendu ne devrait pas planter + try: + html_content = render_email_template(params, "email_template.html.j2") + assert html_content is not None + except Exception as e: + pytest.fail(f"Le rendu ne devrait pas échouer avec des champs manquants: {e}") + + def test_email_params_with_special_characters(self): + """Teste le comportement avec des caractères spéciaux.""" + articles_with_special_chars = [ + { + 'title': "L'article avec des caractères spéciaux: é, à, ç, €, <>&", + 'link': 'https://domain.ntld', + 'summary': 'Résumé avec "guillemets" et \'apostrophes\'', + 'score': '75', + 'published': '2025-01-01T12:00:00', + 'source': 'rss' + } + ] + + params = EmailTemplateParams( + articles=articles_with_special_chars, + keywords=['test'], + threshold=0.5 + ) + + html_content = render_email_template(params, "email_template.html.j2") + + # Vérifier que l'autoescape fonctionne (< doit être échappé en <) + assert "<" in html_content or "<" not in html_content.split("")[1].split("")[0] + + +# ========================= +# Test de simulation manuel +# ========================= + +@pytest.mark.manual +@pytest.mark.skipif(True, reason="Test manuel - nécessite une vraie configuration SMTP") +def test_send_real_email_manual(): + """ + Test manuel pour envoyer un vrai email. + + À exécuter avec: pytest -m manual --no-skip + Nécessite de configurer les vraies variables d'environnement SMTP. + """ + articles = [ + { + 'title': "Test d'intégration - L'IA révolutionne la médecine en 2025", + 'link': 'https://domain.ntld', + 'summary': 'Ceci est un test d\'envoi d\'email depuis les tests d\'intégration', + 'score': '60', + 'published': '2025-01-01T20:30:00', + 'source': 'rss' + }, + ] + + email_params = EmailTemplateParams( + articles=articles, + keywords=['test', 'intégration'], + threshold=0.5 + ) + + # Envoyer réellement l'email + send_watch_articles(email_params) + print("✅ Email de test envoyé avec succès!") diff --git a/web.py b/web.py index d4d14dd..cbf4eeb 100644 --- a/web.py +++ b/web.py @@ -8,12 +8,12 @@ from fastapi import Request from dotenv import load_dotenv -from jinja_filters import register_jinja_filters from typing import Optional import logging +from app.jinja_filters import register_jinja_filters from app.services.models import SourceType from app.db.db import ArticleFTS, get_db @@ -23,9 +23,11 @@ load_dotenv() DB_PATH = os.getenv("DB_PATH", "techno-watch.db") +TEMPLATES_WEB = "app/templates/web" + app = FastAPI() -app.mount("/static", StaticFiles(directory="templates/web"), name="static") -templates = Jinja2Templates(directory="templates/web") +app.mount("/static", StaticFiles(directory=TEMPLATES_WEB), name="static") +templates = Jinja2Templates(directory=TEMPLATES_WEB) register_jinja_filters(templates.env) From 09fda5f1226fe79ab823b8cd7d1362f377a04c80 Mon Sep 17 00:00:00 2001 From: Olivier Duval Date: Mon, 27 Oct 2025 13:32:31 +0100 Subject: [PATCH 05/13] =?UTF-8?q?fix=20db.db=20=3D>=20app.db=20save=5Fnode?= =?UTF-8?q?s=20d=C3=A9placement=20ollama.yml=20=3D>=20docker?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .pre-commit.yaml | 0 app/nodes/save_nodes.py | 5 ++--- ollama.yml => docker/ollama.yml | 5 ++++- 3 files changed, 6 insertions(+), 4 deletions(-) delete mode 100644 .pre-commit.yaml rename ollama.yml => docker/ollama.yml (83%) diff --git a/.pre-commit.yaml b/.pre-commit.yaml deleted file mode 100644 index e69de29..0000000 diff --git a/app/nodes/save_nodes.py b/app/nodes/save_nodes.py index d9e10c4..df96a93 100644 --- a/app/nodes/save_nodes.py +++ b/app/nodes/save_nodes.py @@ -3,10 +3,9 @@ from colorama import Fore from app.core.logger import logger from app.models.states import RSSState +from app.db import save_to_db -def save_articles_node(state: RSSState) -> RSSState: - from db.db import save_to_db - +def save_articles_node(state: RSSState) -> RSSState: logger.info(Fore.LIGHTWHITE_EX + "Sauvegarde des articles résumés en DB") if len(state.summaries) > 0: save_to_db(state.summaries) diff --git a/ollama.yml b/docker/ollama.yml similarity index 83% rename from ollama.yml rename to docker/ollama.yml index bca2681..a44997f 100644 --- a/ollama.yml +++ b/docker/ollama.yml @@ -2,9 +2,12 @@ services: ollama: container_name: ollama image: ollama/ollama + # env_file: + # - ../.env # Chemin relatif vers le .env à la racine # runtime: nvidia environment: + LLM_MODEL: "mistral" NVIDIA_VISIBLE_DEVICES: "all" OLLAMA_KV_CACHE_TYPE: "q4_0" OLLAMA_KEEP_ALIVE: "24h" @@ -23,7 +26,7 @@ services: volumes: - ollama:/root/.ollama restart: unless-stopped - entrypoint: /bin/sh -c "ollama serve & sleep 3 && ollama pull ${LLM_MODEL} && wait" + entrypoint: /bin/sh -c "ollama serve & sleep 3 && ollama pull $${LLM_MODEL} && wait" volumes: ollama: From 1697dae13d1450ffaaeb9f9914ab53049d8fab3c Mon Sep 17 00:00:00 2001 From: Olivier Duval Date: Mon, 27 Oct 2025 13:52:44 +0100 Subject: [PATCH 06/13] =?UTF-8?q?init=20Dockerfile=20app=20ollama.yml=20?= =?UTF-8?q?=C3=A0=20la=20racine=20pour=20le=20.env?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 3 ++- docker/Dockerfile.app | 33 +++++++++++++++++++++++++++++++++ docker/ollama.yml => ollama.yml | 3 ++- techno-watch.yml | 9 +++++++++ 4 files changed, 46 insertions(+), 2 deletions(-) create mode 100644 docker/Dockerfile.app rename docker/ollama.yml => ollama.yml (86%) create mode 100644 techno-watch.yml diff --git a/README.md b/README.md index 55b0a9d..39d5f27 100644 --- a/README.md +++ b/README.md @@ -122,8 +122,9 @@ $ python main_agent.py --debug ## Tests ```bash -$ pytest tests/ +$ pytest tests $ pytest tests/test_sources_ponderation.py -v +$ pytest tests/test_send_articles_email.py -v ``` ## Interface UI pour les articles résumés diff --git a/docker/Dockerfile.app b/docker/Dockerfile.app new file mode 100644 index 0000000..8d30b66 --- /dev/null +++ b/docker/Dockerfile.app @@ -0,0 +1,33 @@ +# Dockerfile pour l'application CLI techno_watch_ia +# docker build -f docker/Dockerfile.app -t techno-watch-ia:latest . +# docker run --env-file .env techno-watch-ia:latest +FROM python:3.11-slim + +# Variables d'environnement pour Python +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + # Configuration uv + UV_SYSTEM_PYTHON=1 + +# Installation de uv +COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv + +# Répertoire de travail +WORKDIR /app + +# Copie des fichiers de dépendances +COPY pyproject.toml ./ +COPY uv.lock* ./ + +# Installation des dépendances avec uv +RUN uv sync --frozen --no-dev + +# Copie du code source +COPY app/ ./app/ +# COPY tests/ ./tests/ + +# Point d'entrée pour lancer l'application +ENTRYPOINT ["python", "-m", "app"] + +# Arguments par défaut (peuvent être surchargés) +CMD [] \ No newline at end of file diff --git a/docker/ollama.yml b/ollama.yml similarity index 86% rename from docker/ollama.yml rename to ollama.yml index a44997f..a0dbd71 100644 --- a/docker/ollama.yml +++ b/ollama.yml @@ -26,7 +26,8 @@ services: volumes: - ollama:/root/.ollama restart: unless-stopped - entrypoint: /bin/sh -c "ollama serve & sleep 3 && ollama pull $${LLM_MODEL} && wait" + entrypoint: /bin/sh -c "ollama serve & sleep 3 && ollama pull ${LLM_MODEL} && wait" + # entrypoint: /bin/sh -c "ollama serve & sleep 3 && ollama pull $${LLM_MODEL} && wait" volumes: ollama: diff --git a/techno-watch.yml b/techno-watch.yml new file mode 100644 index 0000000..d51382a --- /dev/null +++ b/techno-watch.yml @@ -0,0 +1,9 @@ +services: + techno-watch-ia: + build: + context: . + dockerfile: docker/Dockerfile.app + env_file: + - .env + # volumes: + # - ./results:/app/results \ No newline at end of file From 76407e2de60122e3fabbd9b01994158c6efcbc28 Mon Sep 17 00:00:00 2001 From: Olivier Duval Date: Mon, 27 Oct 2025 14:08:57 +0100 Subject: [PATCH 07/13] fix Dockerfile.app + compose techno-watch.yml --- docker/Dockerfile.app | 37 +++++++++++++++++++++++-------------- pyproject.toml | 2 +- techno-watch.yml | 7 ++++--- 3 files changed, 28 insertions(+), 18 deletions(-) diff --git a/docker/Dockerfile.app b/docker/Dockerfile.app index 8d30b66..04495de 100644 --- a/docker/Dockerfile.app +++ b/docker/Dockerfile.app @@ -1,33 +1,42 @@ # Dockerfile pour l'application CLI techno_watch_ia -# docker build -f docker/Dockerfile.app -t techno-watch-ia:latest . +# par le cli docker : docker build -f docker/Dockerfile.app -t techno-watch-ia:latest . +# par le compose : docker compose -f techno-watch.yml build +# run par le cli : # docker run --env-file .env techno-watch-ia:latest -FROM python:3.11-slim +# docker run --env-file .env -v $(pwd)/data:/app/data -w /app techno-watch-ia:latest +# run par le compose : +# docker compose -f techno-watch.yml run app + +FROM python:3.11-slim AS builder -# Variables d'environnement pour Python +# Variables d'environnement pour Python et uv ENV PYTHONUNBUFFERED=1 \ PYTHONDONTWRITEBYTECODE=1 \ - # Configuration uv UV_SYSTEM_PYTHON=1 -# Installation de uv COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv -# Répertoire de travail WORKDIR /app -# Copie des fichiers de dépendances -COPY pyproject.toml ./ -COPY uv.lock* ./ +COPY pyproject.toml uv.lock* ./ -# Installation des dépendances avec uv RUN uv sync --frozen --no-dev -# Copie du code source +# Stage 2: Runtime +FROM python:3.11-slim + +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 + +WORKDIR /app + +COPY --from=builder /app/.venv /app/.venv COPY app/ ./app/ -# COPY tests/ ./tests/ -# Point d'entrée pour lancer l'application +ENV PATH="/app/.venv/bin:$PATH" + +VOLUME ["/app/data"] + ENTRYPOINT ["python", "-m", "app"] -# Arguments par défaut (peuvent être surchargés) CMD [] \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 582d247..d9ce358 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ dependencies = [ "torchvision>=0.22.0", "beautifulsoup4>=4.13.5", "colorama>=0.4.6", - "dotenv>=0.9.9", + "python-dotenv>=1.0.0", "faiss-cpu>=1.12.0", "feedparser>=6.0.11", "langchain-community>=0.3.29", diff --git a/techno-watch.yml b/techno-watch.yml index d51382a..11ad64b 100644 --- a/techno-watch.yml +++ b/techno-watch.yml @@ -1,9 +1,10 @@ services: - techno-watch-ia: + app: build: context: . dockerfile: docker/Dockerfile.app + image: techno-watch-ia:latest env_file: - .env - # volumes: - # - ./results:/app/results \ No newline at end of file + volumes: + - ./data:/app/data \ No newline at end of file From 7f34ccb5b8922a6c596917b74cb854aa486b402b Mon Sep 17 00:00:00 2001 From: Olivier Duval Date: Mon, 27 Oct 2025 15:17:37 +0100 Subject: [PATCH 08/13] fix read_opml.py => app --- app/nodes/utils_fetch_nodes.py | 2 +- read_opml.py => app/read_opml.py | 0 docker/Dockerfile.app | 6 ++++++ 3 files changed, 7 insertions(+), 1 deletion(-) rename read_opml.py => app/read_opml.py (100%) diff --git a/app/nodes/utils_fetch_nodes.py b/app/nodes/utils_fetch_nodes.py index 7cef07e..3ae68af 100644 --- a/app/nodes/utils_fetch_nodes.py +++ b/app/nodes/utils_fetch_nodes.py @@ -29,7 +29,7 @@ def get_rss_urls(): Obtient la liste des URL RSS à traiter à partir des variables d'environnement. Le .env ne contient que des types string et au format JSON """ - from read_opml import parse_opml_to_rss_list + from app.read_opml import parse_opml_to_rss_list from app.services.models import Source, SourceType OPML_FILE = get_environment_variable("OPML_FILE", "my.opml") diff --git a/read_opml.py b/app/read_opml.py similarity index 100% rename from read_opml.py rename to app/read_opml.py diff --git a/docker/Dockerfile.app b/docker/Dockerfile.app index 04495de..fd53060 100644 --- a/docker/Dockerfile.app +++ b/docker/Dockerfile.app @@ -1,11 +1,17 @@ # Dockerfile pour l'application CLI techno_watch_ia +# # par le cli docker : docker build -f docker/Dockerfile.app -t techno-watch-ia:latest . # par le compose : docker compose -f techno-watch.yml build +# # run par le cli : # docker run --env-file .env techno-watch-ia:latest # docker run --env-file .env -v $(pwd)/data:/app/data -w /app techno-watch-ia:latest # run par le compose : # docker compose -f techno-watch.yml run app +# +# entrer dans le container en surchargeant l'entrypoint mis dans le Dockerfile : +# docker compose -f techno-watch.yml run --rm --entrypoint bash app +# docker run --rm -it --entrypoint bash techno-watch-ia -v ./data:/app/data FROM python:3.11-slim AS builder From c6bad631aeedfaeedf5d791c0f589fe44485c181 Mon Sep 17 00:00:00 2001 From: Olivier Duval Date: Mon, 27 Oct 2025 15:19:50 +0100 Subject: [PATCH 09/13] comment Dockerfile.app pour construction de l'image techno-watch-ia --- docker/Dockerfile.app | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile.app b/docker/Dockerfile.app index fd53060..bdc1073 100644 --- a/docker/Dockerfile.app +++ b/docker/Dockerfile.app @@ -1,5 +1,6 @@ -# Dockerfile pour l'application CLI techno_watch_ia -# +# Dockerfile pour l'application CLI techno_watch_ia, en multi stages +# utilisation d'uv pour l'installation des packages +# # par le cli docker : docker build -f docker/Dockerfile.app -t techno-watch-ia:latest . # par le compose : docker compose -f techno-watch.yml build # @@ -13,6 +14,7 @@ # docker compose -f techno-watch.yml run --rm --entrypoint bash app # docker run --rm -it --entrypoint bash techno-watch-ia -v ./data:/app/data +# Stage 1 FROM python:3.11-slim AS builder # Variables d'environnement pour Python et uv From 714d2876972e7ab03899ee504a4cb0962af189d6 Mon Sep 17 00:00:00 2001 From: Olivier Duval Date: Mon, 27 Oct 2025 15:24:44 +0100 Subject: [PATCH 10/13] fix package coloram docker/requirements-web.txt image techno-watch-web --- docker/requirements-web.txt | 4 +++- web.yml | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docker/requirements-web.txt b/docker/requirements-web.txt index b4c6604..d71f425 100644 --- a/docker/requirements-web.txt +++ b/docker/requirements-web.txt @@ -19,4 +19,6 @@ pytz>=2025.2 pydantic>=2.11.7 # Multipart forms (pour FastAPI) -python-multipart>=0.0.6 \ No newline at end of file +python-multipart>=0.0.6 + +colorama>=0.4.6 \ No newline at end of file diff --git a/web.yml b/web.yml index 3a0f2f8..0816a71 100644 --- a/web.yml +++ b/web.yml @@ -1,3 +1,5 @@ +# build image techno-watch-web : docker compose -f web.yml build +# lancement : docker compose -f web.yml up services: fastapi-app: build: From 1cd43d104bea1c293d1df2680c4c974c03b214de Mon Sep 17 00:00:00 2001 From: Olivier Duval Date: Mon, 27 Oct 2025 16:40:53 +0100 Subject: [PATCH 11/13] ruff format app ajout package ruff --dev --- app/__init__.py | 14 +-- app/__main__.py | 2 +- app/core/__init__.py | 9 +- app/core/logger.py | 13 +- app/core/utils.py | 12 +- app/db/__init__.py | 7 +- app/db/db.py | 142 +++++++++++++--------- app/jinja_filters.py | 63 ++++++---- app/main_agent.py | 87 +++++++++----- app/models/__init__.py | 2 +- app/models/article.py | 6 +- app/models/emails.py | 3 +- app/models/states.py | 11 +- app/nodes/__init__.py | 14 ++- app/nodes/fetch_nodes.py | 92 ++++++++------ app/nodes/filter_nodes.py | 21 ++-- app/nodes/output_nodes.py | 4 +- app/nodes/save_nodes.py | 4 +- app/nodes/send_nodes.py | 10 +- app/nodes/summarize_nodes.py | 32 +++-- app/nodes/utils_fetch_nodes.py | 81 ++++++++----- app/read_opml.py | 1 + app/send_articles_email.py | 64 ++++++---- app/services/decorators.py | 36 ++++-- app/services/factory_fetcher.py | 27 +++-- app/services/fetchers/__init__.py | 4 +- app/services/fetchers/base_fetcher.py | 3 +- app/services/fetchers/bluesky_fetcher.py | 145 +++++++++++++---------- app/services/fetchers/reedit_fetcher.py | 70 ++++++----- app/services/fetchers/rss_fetcher.py | 22 ++-- app/services/model_service.py | 13 +- app/services/models.py | 12 +- app/services/sources_ponderation.py | 141 +++++++++++++--------- app/services/utils_fetchers.py | 16 +-- uv.lock | 15 +-- 35 files changed, 720 insertions(+), 478 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index e9e8a4d..86151a3 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -1,13 +1,13 @@ # from core import measure_time, argscli, logger from .core.logger import logger -from .core.utils import measure_time #, get_environment_variable +from .core.utils import measure_time # , get_environment_variable from .models.emails import EmailTemplateParams from .models.states import RSSState __all__ = [ - 'logger', - 'measure_time', - # 'argscli', - 'EmailTemplateParams', - 'RSSState' -] \ No newline at end of file + "logger", + "measure_time", + # 'argscli', + "EmailTemplateParams", + "RSSState", +] diff --git a/app/__main__.py b/app/__main__.py index 5dde2c5..5b596fc 100644 --- a/app/__main__.py +++ b/app/__main__.py @@ -1,4 +1,4 @@ from .main_agent import main # ou votre fonction principale if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/app/core/__init__.py b/app/core/__init__.py index 3b81b5b..e5c88d8 100644 --- a/app/core/__init__.py +++ b/app/core/__init__.py @@ -2,9 +2,10 @@ # from .utils import argscli from .utils import measure_time, get_environment_variable from .logger import logger + __all__ = [ - 'measure_time', - 'logger', + "measure_time", + "logger", # 'argscli', - 'get_environment_variable' -] \ No newline at end of file + "get_environment_variable", +] diff --git a/app/core/logger.py b/app/core/logger.py index 0957a81..6c419e0 100644 --- a/app/core/logger.py +++ b/app/core/logger.py @@ -5,9 +5,11 @@ logging.basicConfig(level=logging.INFO) + def print_color(color, text): print(color + text) + # ========================= # Formatter coloré # ========================= @@ -26,10 +28,10 @@ def format(self, record): record.levelname = levelname_color return super().format(record) + def setup_logger(level=logging.INFO): formatter = ColorFormatter( - fmt="%(asctime)s - %(levelname)s - %(message)s", - datefmt="%d/%m/%Y %H:%M:%S" + fmt="%(asctime)s - %(levelname)s - %(message)s", datefmt="%d/%m/%Y %H:%M:%S" ) handler = logging.StreamHandler() handler.setFormatter(formatter) @@ -40,11 +42,14 @@ def setup_logger(level=logging.INFO): logger.propagate = False return logger + logger = setup_logger() + def count_by_type_articles(title, articles_by_source, color=Fore.LIGHTYELLOW_EX): from collections import Counter - source_counts = Counter(item['source'].value for item in articles_by_source) + + source_counts = Counter(item["source"].value for item in articles_by_source) print_color(color, "=" * 60) print_color(color, f"{title} {source_counts}") - print_color(color, "=" * 60) \ No newline at end of file + print_color(color, "=" * 60) diff --git a/app/core/utils.py b/app/core/utils.py index 14771dc..3841645 100644 --- a/app/core/utils.py +++ b/app/core/utils.py @@ -13,19 +13,24 @@ # Couleurs # ========================= + def parse_cli_args(): parser = argparse.ArgumentParser(description="Agent RSS avec résumés LLM") - parser.add_argument("--debug", action="store_true", help="Active le mode debug détaillé") + parser.add_argument( + "--debug", action="store_true", help="Active le mode debug détaillé" + ) return parser.parse_args() + def configure_logging_from_args(): args = parse_cli_args() level = logging.DEBUG if args.debug else logging.INFO return setup_logger(level=level), args + def measure_time(func): @wraps(func) - def wrapper(*args, **kwargs): + def wrapper(*args, **kwargs): start_time = time.time() result = func(*args, **kwargs) end_time = time.time() @@ -36,6 +41,7 @@ def wrapper(*args, **kwargs): return wrapper -def get_environment_variable(key, default = None): + +def get_environment_variable(key, default=None): load_dotenv() return os.getenv(key, default) diff --git a/app/db/__init__.py b/app/db/__init__.py index 6bcedc5..ff83673 100644 --- a/app/db/__init__.py +++ b/app/db/__init__.py @@ -1,9 +1,4 @@ from .db import init_db, save_to_db, DB_PATH from .db import read_articles -__all__=[ - 'init_db', - 'save_to_db', - 'DB_PATH', - 'read_articles' -] \ No newline at end of file +__all__ = ["init_db", "save_to_db", "DB_PATH", "read_articles"] diff --git a/app/db/db.py b/app/db/db.py index 21b94e1..74abf43 100644 --- a/app/db/db.py +++ b/app/db/db.py @@ -22,43 +22,53 @@ logger = logging.getLogger(__name__) Base = declarative_base() + + # # Modèles # /!\ Après le declarative_base() # class Article(Base): """Modèle entité BDD""" - __tablename__ = 'articles' + + __tablename__ = "articles" id = Column(Integer, primary_key=True) - dt_created = Column(DateTime, - server_default=func.now(), - default=lambda: datetime.now(timezone.utc), - nullable=False, - index=True) - dt_updated = Column(DateTime, - onupdate=lambda: datetime.now(timezone.utc), # Remplace utcnow() - default=lambda: datetime.now(timezone.utc), - nullable=False) + dt_created = Column( + DateTime, + server_default=func.now(), + default=lambda: datetime.now(timezone.utc), + nullable=False, + index=True, + ) + dt_updated = Column( + DateTime, + onupdate=lambda: datetime.now(timezone.utc), # Remplace utcnow() + default=lambda: datetime.now(timezone.utc), + nullable=False, + ) title = Column(String, nullable=False) link = Column(String, nullable=False) summary = Column(Text, nullable=False) score = Column(String, nullable=False) - published = Column(String, nullable=False, index=True) + published = Column(String, nullable=False, index=True) source = Column( SQLAlchemyEnum(SourceType), nullable=False, default=SourceType.RSS, # Optionnel : valeur par défaut - index=True # Optionnel : index pour les requêtes + index=True, # Optionnel : index pour les requêtes ) + # Evènement pour màj de dt_updated - /!\ après la déclaration du modèle -@event.listens_for(Article, 'before_update') -def update_timestamp(mapper, connection, target): +@event.listens_for(Article, "before_update") +def update_timestamp(mapper, connection, target): target.dt_updated = datetime.now(timezone.utc) + class ArticleFTS: """Modèle pour la table FTS5 Full Text Search""" - __tablename__ = 'articles_fts' + + __tablename__ = "articles_fts" @classmethod def execute_statement(cls, conn, statement): @@ -68,17 +78,19 @@ def execute_statement(cls, conn, statement): @classmethod def create_trigger_if_not_exists(cls, conn, trigger_name, trigger_sql): # Vérifier si le trigger existe déjà - result = conn.execute(text(f""" + result = conn.execute( + text(f""" SELECT name FROM sqlite_master WHERE type='trigger' AND name='{trigger_name}' - """)).fetchone() + """) + ).fetchone() if not result: conn.execute(text(trigger_sql)) conn.commit() @classmethod - def init_table(cls, engine): + def init_table(cls, engine): """Crée la table FTS5 manuellement.""" triggers = { "sync_article_fts": f""" @@ -101,11 +113,13 @@ def init_table(cls, engine): BEGIN DELETE FROM {cls.__tablename__} WHERE article_id = old.id; END; - """ - } - - with engine.connect() as conn: - cls.execute_statement(conn, text(f""" + """, + } + + with engine.connect() as conn: + cls.execute_statement( + conn, + text(f""" CREATE VIRTUAL TABLE IF NOT EXISTS {cls.__tablename__} USING fts5( article_id, title, @@ -113,9 +127,10 @@ def init_table(cls, engine): tokenize='unicode61', prefix='2,3' ); - """)) + """), + ) for name, sql in triggers.items(): - cls.create_trigger_if_not_exists(conn, name, sql) + cls.create_trigger_if_not_exists(conn, name, sql) @classmethod def insert(cls, session, title, content, published): @@ -125,7 +140,7 @@ def insert(cls, session, title, content, published): INSERT OR IGNORE INTO {cls.__tablename__} (title, content, published) VALUES (:title, :content, :published) """), - {'title': title, 'content': content, 'published': published} + {"title": title, "content": content, "published": published}, ) @classmethod @@ -137,7 +152,7 @@ def bulk_insert(cls, session, articles): INSERT OR IGNORE INTO {cls.__tablename__} (title, content, published) VALUES (:title, :content, :published) """), - articles # Liste de dicts [{'title': '...', 'content': '...', 'published': '...'}, ...] + articles, # Liste de dicts [{'title': '...', 'content': '...', 'published': '...'}, ...] ) @classmethod @@ -158,8 +173,8 @@ def search(cls, session, query, date_min=None, date_max=None, limit=10): """ # Les indices des 2 champs hightlights de la table articles_fts IDX_TITLE_TABLE = 1 - IDX_CONTENT_TABLE = 2 - + IDX_CONTENT_TABLE = 2 + # Base SQL pour le CTE base_sql = f""" WITH ranked AS ( @@ -196,7 +211,7 @@ def search(cls, session, query, date_min=None, date_max=None, limit=10): FROM ranked JOIN {Article.__tablename__} a on a.id = ranked.article_id """ - + if where_clauses: base_sql += " WHERE " + " AND ".join(where_clauses) logger.info(f"where {base_sql}") @@ -207,12 +222,12 @@ def search(cls, session, query, date_min=None, date_max=None, limit=10): """ logger.info(f"SQL exécuté: {base_sql} avec {params}") - + results = session.execute(text(base_sql), params) results_as_dict = results.mappings().all() logger.debug(f"results : {results_as_dict}") - + return results_as_dict # return results.fetchall() @@ -274,27 +289,33 @@ def search_with_bm25(cls, session, query, date_min=None, date_max=None, limit=10 logger.debug(f"SQL exécuté: {base_sql} avec {params}") return session.execute(text(base_sql), params).fetchall() + # # Configuration SQL Alchemy -# -SQLALCHEMY_DB = f'sqlite:///{DB_PATH}' +# +SQLALCHEMY_DB = f"sqlite:///{DB_PATH}" engine = create_engine( SQLALCHEMY_DB, - connect_args={"check_same_thread": False}, # Nécessaire pour SQLite avec FastAPI et le multi-threads - echo=True # Affiche les requêtes SQL (optionnel, pour le debug)) + connect_args={ + "check_same_thread": False + }, # Nécessaire pour SQLite avec FastAPI et le multi-threads + echo=True, # Affiche les requêtes SQL (optionnel, pour le debug)) ) + def init_db(): """Initialise la base de données SQLite.""" Base.metadata.create_all(engine) ArticleFTS.init_table(engine) print("Table 'articles' initialisée avec succès !") + SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) + # # Fonctions utilitaires DB -# +# @contextmanager def get_db(): """Générateur de session SQLAlchemy avec gestion automatique pour le close().""" @@ -304,26 +325,29 @@ def get_db(): finally: session.close() + def read_articles(date: str = None): - """Lit les articles résumés qui ont été retenus pour la veille techno""" + """Lit les articles résumés qui ont été retenus pour la veille techno""" with get_db() as session: if date: - articles = session.query(Article).filter(Article.published.like(f"%{date}%")).all() + articles = ( + session.query(Article).filter(Article.published.like(f"%{date}%")).all() + ) else: articles = session.query(Article).order_by(Article.published.desc()).all() return articles + def _validate_and_get_articles_summaries(summaries): for item in summaries: logging.info(f"** item summaries : {item}\n") validated_articles = [ArticleModel(**item) for item in summaries] - articles_data = [ - article.model_dump() for article in validated_articles - ] + articles_data = [article.model_dump() for article in validated_articles] logger.info(f"Articles validés pour insertion: {articles_data}") return articles_data + def save_to_db(summaries: list[dict]): """ Sauvegarde les articles en base avec validation Pydantic avec insertion en bulk. @@ -335,23 +359,25 @@ def save_to_db(summaries: list[dict]): Raises: ValueError: Si la validation Pydantic échoue Exception: Pour les erreurs de base de données - """ - with get_db() as session: - try: + """ + with get_db() as session: + try: existing = session.query(Article.title, Article.published).all() existing_pairs = {(title, published) for title, published in existing} new_articles = [ - item for item in summaries + item + for item in summaries if (item["title"], item["published"]) not in existing_pairs ] if new_articles: - logger.info(f"Nombre de nouveaux articles {len(new_articles)}") - articles_data = _validate_and_get_articles_summaries(new_articles) + logger.info(f"Nombre de nouveaux articles {len(new_articles)}") + articles_data = _validate_and_get_articles_summaries(new_articles) session.bulk_insert_mappings(Article, articles_data) - session.commit() + session.commit() except Exception as e: session.rollback() - raise e + raise e + def search_fts(keywords: str): with get_db() as session: @@ -361,17 +387,17 @@ def search_fts(keywords: str): # Recherche avec filtre date result2 = ArticleFTS.search( - session, - "cybersécurité", - date_min="2025-01-01", - date_max="2025-12-31" + session, "cybersécurité", date_min="2025-01-01", date_max="2025-12-31" ) logger.info(f"Résultats FTS: {result2}") # Recherche depuis une date - result3 = ArticleFTS.search(session, keywords or "docker", date_min="2025-09-01") + result3 = ArticleFTS.search( + session, keywords or "docker", date_min="2025-09-01" + ) logger.info(f"Résultats FTS: {result3}") -if __name__ == "__main__": - articles = read_articles() - print(f"Nombre d'articles en base: {len(articles)}") \ No newline at end of file + +if __name__ == "__main__": + articles = read_articles() + print(f"Nombre d'articles en base: {len(articles)}") diff --git a/app/jinja_filters.py b/app/jinja_filters.py index 272090e..7c863e6 100644 --- a/app/jinja_filters.py +++ b/app/jinja_filters.py @@ -10,10 +10,11 @@ logging.basicConfig(level=logging.INFO) + # # Filtres pour les templates jinja2 mail ou web # champ | filtre -# +# def format_date(value): """filtre jinga2 pour formatage de la date en FR""" if isinstance(value, str): @@ -21,29 +22,34 @@ def format_date(value): return date_obj.strftime("%d/%m/%Y") return value.strftime("%d/%m/%Y") + def format_local_datetime(utc_datetime, tz: str = "Europe/Paris"): - """filtre de conversion UTC -> heure Paris""" + """filtre de conversion UTC -> heure Paris""" import pytz + if not utc_datetime: return "N/A" # Convertit UTC → Europe/Paris local_tz = pytz.timezone(tz) local_datetime = utc_datetime.replace(tzinfo=timezone.utc).astimezone(local_tz) - return local_datetime.strftime('%d/%m/%Y à %H:%M') + return local_datetime.strftime("%d/%m/%Y à %H:%M") + def nl2br(value): """Convertit les retours chariots en
.""" if value is None: return "" - return Markup(value.replace('\n', '
\n')) + return Markup(value.replace("\n", "
\n")) + @lru_cache(maxsize=32) def get_svg_base64(source: str) -> str: - """cache des icones SVG en base64 pour les emails.""" + """cache des icones SVG en base64 pour les emails.""" icon_path = Path(__file__).parent / "templates" / "web" / "icons" / f"{source}.svg" with open(icon_path, "rb") as f: return base64.b64encode(f.read()).decode("utf-8") + def icon_html(source: SourceType, size: int = 24, email: bool = False) -> str: """Génère le HTML d'une icône SVG pour le web ou les emails. @@ -51,47 +57,54 @@ def icon_html(source: SourceType, size: int = 24, email: bool = False) -> str: source: Type de source ("bluesky", "reddit", "rss", "linkedin", "twitter"). size: Taille en pixels (défaut: 24). email: Si True, retourne une balise avec SVG en base64. - """ - logging.debug(f"icon_html: source={source}, size={size}, email={email}") + """ + logging.debug(f"icon_html: source={source}, size={size}, email={email}") icon_path = Path(__file__).parent / "templates" / "web" / "icons" / f"{source}.svg" FALLBACK_EMOJIS = { "rss": "📡", "bluesky": "🐦", - "reddit": "👽", + "reddit": "👽", "linkedin": "💼", "twitter": "🐦", } # logging.info(f"icon_html: icon_path={icon_path}") - - if not icon_path.exists(): # fallback si le chemin n'existe pas + + if not icon_path.exists(): # fallback si le chemin n'existe pas logging.error(f"Le chemin des icônes n'existe pas: {icon_path}") if email: - return Markup(f'{FALLBACK_EMOJIS.get(source, "🔗")}') + return Markup( + f'{FALLBACK_EMOJIS.get(source, "🔗")}' + ) else: - return Markup(f'{FALLBACK_EMOJIS.get(source, "🔗")}') - - if email: - logging.debug(f"icon_html: icon_path={icon_path}") + return Markup( + f'{FALLBACK_EMOJIS.get(source, "🔗")}' + ) + + if email: + logging.debug(f"icon_html: icon_path={icon_path}") with open(icon_path, "rb") as f: svg_base64 = base64.b64encode(f.read()).decode("utf-8") - return Markup(f'{source}') + return Markup( + f'{source}' + ) - html = f''' -