From 659ef97fcad6b9c583dcc79f8260ef485af4ae1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dawid=20P=C5=82ocki?= Date: Sat, 20 Apr 2024 16:29:37 +0200 Subject: [PATCH 01/10] Start writing debug.py --- debug.py | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 debug.py diff --git a/debug.py b/debug.py new file mode 100644 index 0000000..af5fc5d --- /dev/null +++ b/debug.py @@ -0,0 +1,7 @@ +from podcast_downloader.rss import load_feed + + +rss_source_link = 'https://feed.theskepticsguide.org/feed/rss.aspx' +feed = load_feed(rss_source_link) + +print(feed) \ No newline at end of file From 944ec13a61588869944a39cfe6685b72842ed9ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dawid=20P=C5=82ocki?= Date: Sat, 20 Apr 2024 16:45:52 +0200 Subject: [PATCH 02/10] Loading config file in the debug.py --- debug.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/debug.py b/debug.py index af5fc5d..b7aadd6 100644 --- a/debug.py +++ b/debug.py @@ -1,7 +1,10 @@ +from podcast_downloader.parameters import load_configuration_file from podcast_downloader.rss import load_feed -rss_source_link = 'https://feed.theskepticsguide.org/feed/rss.aspx' +config = load_configuration_file('a.json') +podcast_config = config['podcasts'][0] +rss_source_link = podcast_config['rss_link'] feed = load_feed(rss_source_link) -print(feed) \ No newline at end of file +print(feed) From dca88d054af1ec7f8685dd0de2c2429fe9ed8ebe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dawid=20P=C5=82ocki?= Date: Mon, 22 Apr 2024 21:55:28 +0200 Subject: [PATCH 03/10] Add loading files from dictionary --- debug.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/debug.py b/debug.py index b7aadd6..9edf9c9 100644 --- a/debug.py +++ b/debug.py @@ -1,10 +1,22 @@ +import os +from podcast_downloader.downloaded import get_downloaded_files, get_extensions_checker from podcast_downloader.parameters import load_configuration_file from podcast_downloader.rss import load_feed -config = load_configuration_file('a.json') -podcast_config = config['podcasts'][0] -rss_source_link = podcast_config['rss_link'] +config = load_configuration_file("a.json") +podcast_config = config["podcasts"][0] +rss_source_link = podcast_config["rss_link"] feed = load_feed(rss_source_link) -print(feed) + +rss_podcast_extensions = podcast_config.get("podcast_extensions", ".mp3") +rss_source_path = os.path.expanduser(podcast_config["path"]) + +downloaded_files = list( + get_downloaded_files( + get_extensions_checker(rss_podcast_extensions), rss_source_path + ) +) + +print(feed, downloaded_files) From 700d03649220318d7f74943e75fbfa996c752f15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dawid=20P=C5=82ocki?= Date: Wed, 24 Apr 2024 18:18:40 +0200 Subject: [PATCH 04/10] Transform the feed --- debug.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/debug.py b/debug.py index 9edf9c9..36f35fb 100644 --- a/debug.py +++ b/debug.py @@ -1,7 +1,9 @@ +from functools import partial import os from podcast_downloader.downloaded import get_downloaded_files, get_extensions_checker from podcast_downloader.parameters import load_configuration_file -from podcast_downloader.rss import load_feed +from podcast_downloader.rss import build_only_allowed_filter_for_link_data, flatten_rss_links_data, get_raw_rss_entries_from_feed, load_feed +from podcast_downloader.utils import compose config = load_configuration_file("a.json") @@ -10,13 +12,23 @@ feed = load_feed(rss_source_link) -rss_podcast_extensions = podcast_config.get("podcast_extensions", ".mp3") +rss_podcast_extensions = podcast_config.get("podcast_extensions", {".mp3": "audio/mpeg"}) rss_source_path = os.path.expanduser(podcast_config["path"]) + +allow_link_types = list(set(rss_podcast_extensions.values())) + +all_feed_entries = compose( + list, + partial(filter, build_only_allowed_filter_for_link_data(allow_link_types)), + flatten_rss_links_data, + get_raw_rss_entries_from_feed, +)(feed) + downloaded_files = list( get_downloaded_files( get_extensions_checker(rss_podcast_extensions), rss_source_path ) ) -print(feed, downloaded_files) +print(all_feed_entries, downloaded_files) From 3bef2cb3e6d8185de68371bef324627b3cecaacb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dawid=20P=C5=82ocki?= Date: Thu, 25 Apr 2024 20:45:30 +0200 Subject: [PATCH 05/10] Debug: Get list of files from the feed --- debug.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/debug.py b/debug.py index 36f35fb..5e4de34 100644 --- a/debug.py +++ b/debug.py @@ -1,8 +1,16 @@ from functools import partial import os +from podcast_downloader.__main__ import get_system_file_name_limit from podcast_downloader.downloaded import get_downloaded_files, get_extensions_checker from podcast_downloader.parameters import load_configuration_file -from podcast_downloader.rss import build_only_allowed_filter_for_link_data, flatten_rss_links_data, get_raw_rss_entries_from_feed, load_feed +from podcast_downloader.rss import ( + build_only_allowed_filter_for_link_data, + file_template_to_file_name, + flatten_rss_links_data, + get_raw_rss_entries_from_feed, + limit_file_name, + load_feed, +) from podcast_downloader.utils import compose @@ -11,8 +19,9 @@ rss_source_link = podcast_config["rss_link"] feed = load_feed(rss_source_link) - -rss_podcast_extensions = podcast_config.get("podcast_extensions", {".mp3": "audio/mpeg"}) +rss_podcast_extensions = podcast_config.get( + "podcast_extensions", {".mp3": "audio/mpeg"} +) rss_source_path = os.path.expanduser(podcast_config["path"]) @@ -31,4 +40,12 @@ ) ) -print(all_feed_entries, downloaded_files) +to_name_function = partial(file_template_to_file_name, "%file_name%.%file_extension%") +file_length_limit = get_system_file_name_limit(podcast_config) +to_real_podcast_file_name = compose( + partial(limit_file_name, file_length_limit), to_name_function +) + +all_feed_files = list(map(to_real_podcast_file_name, all_feed_entries))[::-1] + +print(all_feed_files, downloaded_files) From dfdcfd4fa6fa8f8ab560e20008c885574d629a97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dawid=20P=C5=82ocki?= Date: Thu, 25 Apr 2024 20:48:22 +0200 Subject: [PATCH 06/10] Debug: Add leading the template directory from configuration --- debug.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/debug.py b/debug.py index 5e4de34..977995a 100644 --- a/debug.py +++ b/debug.py @@ -22,6 +22,9 @@ rss_podcast_extensions = podcast_config.get( "podcast_extensions", {".mp3": "audio/mpeg"} ) +rss_podcast_file_name_template = podcast_config.get( + "file_name_template", "%file_name%.%file_extension%" +) rss_source_path = os.path.expanduser(podcast_config["path"]) @@ -40,7 +43,7 @@ ) ) -to_name_function = partial(file_template_to_file_name, "%file_name%.%file_extension%") +to_name_function = partial(file_template_to_file_name, rss_podcast_file_name_template) file_length_limit = get_system_file_name_limit(podcast_config) to_real_podcast_file_name = compose( partial(limit_file_name, file_length_limit), to_name_function From c449e9d67c14159ec251577a341d4b943692a48c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dawid=20P=C5=82ocki?= Date: Fri, 26 Apr 2024 20:08:24 +0200 Subject: [PATCH 07/10] Add finding new link --- debug.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/debug.py b/debug.py index 977995a..3d0f074 100644 --- a/debug.py +++ b/debug.py @@ -5,6 +5,7 @@ from podcast_downloader.parameters import load_configuration_file from podcast_downloader.rss import ( build_only_allowed_filter_for_link_data, + build_only_new_entities, file_template_to_file_name, flatten_rss_links_data, get_raw_rss_entries_from_feed, @@ -50,5 +51,12 @@ ) all_feed_files = list(map(to_real_podcast_file_name, all_feed_entries))[::-1] +last_downloaded_file = downloaded_files[-1] -print(all_feed_files, downloaded_files) +download_limiter_function = partial( + build_only_new_entities(to_name_function), last_downloaded_file +) + +missing_files_links = compose(list, download_limiter_function)(all_feed_entries) + +print(missing_files_links) From c312d13f2dba9814d69fe8b4de07dfb6a9507a92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dawid=20P=C5=82ocki?= Date: Mon, 6 May 2024 18:58:08 +0200 Subject: [PATCH 08/10] Debug: Add code for proper filter out all files which are outside the feed and sort them acording to the feed --- debug.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/debug.py b/debug.py index 3d0f074..f5de7c4 100644 --- a/debug.py +++ b/debug.py @@ -51,7 +51,8 @@ ) all_feed_files = list(map(to_real_podcast_file_name, all_feed_entries))[::-1] -last_downloaded_file = downloaded_files[-1] +only_files_from_feed_sorted = [feed for feed in all_feed_files if feed in downloaded_files] +last_downloaded_file = only_files_from_feed_sorted[-1] download_limiter_function = partial( build_only_new_entities(to_name_function), last_downloaded_file From b8af1b55ad80e09ecb04b5bff8725143460e62c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dawid=20P=C5=82ocki?= Date: Tue, 7 May 2024 23:09:51 +0200 Subject: [PATCH 09/10] Move debug.py file into scripts/find_missing_files.py --- debug.py => scripts /find_missing_files.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename debug.py => scripts /find_missing_files.py (100%) diff --git a/debug.py b/scripts /find_missing_files.py similarity index 100% rename from debug.py rename to scripts /find_missing_files.py From 7279c2940213b4d0f3f0f29b7635671237b5544f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dawid=20P=C5=82ocki?= Date: Wed, 8 May 2024 21:29:06 +0200 Subject: [PATCH 10/10] Finish debug feed script --- .../debug_feed.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) rename scripts /find_missing_files.py => scripts/debug_feed.py (82%) diff --git a/scripts /find_missing_files.py b/scripts/debug_feed.py similarity index 82% rename from scripts /find_missing_files.py rename to scripts/debug_feed.py index f5de7c4..f0c3993 100644 --- a/scripts /find_missing_files.py +++ b/scripts/debug_feed.py @@ -51,7 +51,9 @@ ) all_feed_files = list(map(to_real_podcast_file_name, all_feed_entries))[::-1] -only_files_from_feed_sorted = [feed for feed in all_feed_files if feed in downloaded_files] +only_files_from_feed_sorted = [ + feed for feed in all_feed_files if feed in downloaded_files +] last_downloaded_file = only_files_from_feed_sorted[-1] download_limiter_function = partial( @@ -60,4 +62,13 @@ missing_files_links = compose(list, download_limiter_function)(all_feed_entries) -print(missing_files_links) +for feed in all_feed_entries: + feed_file = to_real_podcast_file_name(feed) + + status = ( + "to-download" + if feed_file in missing_files_links + else ("downloaded" if feed_file in only_files_from_feed_sorted else "ignored") + ) + + print(feed.title + "\t" + feed_file + "\t" + status)