From ce06c4e4c54a244baeb7e74ac32d9f3237289fd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABl=20Honorez?= Date: Sun, 31 Dec 2023 15:30:16 +0100 Subject: [PATCH 01/13] init commit --- Post_RSS_on_SubStack.py | 207 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 207 insertions(+) create mode 100644 Post_RSS_on_SubStack.py diff --git a/Post_RSS_on_SubStack.py b/Post_RSS_on_SubStack.py new file mode 100644 index 0000000..2a76dda --- /dev/null +++ b/Post_RSS_on_SubStack.py @@ -0,0 +1,207 @@ +import asyncio +import argparse +import requests +import feedparser +import io +import html +import datetime +import logging +import os +import re +from logging.handlers import RotatingFileHandler +import locale +import random + +from substack import Api +from substack.post import Post + +LOG = logging.getLogger('bot') +LOG_PATTERN = logging.Formatter('%(asctime)s:%(levelname)s: [%(filename)s] %(message)s') + + +locale.setlocale(locale.LC_TIME, 'fr_FR') + + +def setuplogger(): + + conf_filename = None + + steam_handler = logging.StreamHandler() + steam_handler.setFormatter(LOG_PATTERN) + steam_handler.setLevel(logging.DEBUG) + + def setup_logger(logger_name, file_name=None, add_steam=False): + file_name = file_name or logger_name + log_filename = f"{file_name}.log" + + logger = logging.getLogger(logger_name) + logger.setLevel(logging.DEBUG) + file_handler = RotatingFileHandler(log_filename, "a", 1000000, 1) + file_handler.setFormatter(LOG_PATTERN) + logger.addHandler(file_handler) + if add_steam: + logger.addHandler(steam_handler) + + setup_logger("bot", conf_filename, True) + +class RSSfeed(): + def __init__(self, url, yt=False): + self.url = url + self.youtube = yt + +class SubStackTask: + def __init__(self, login, password, account, feeds): + self.api = Api( + email=login, + password=password, + publication_url=account, + ) + + self.user_id = self.api.get_user_id() + self.feeds = feeds + + + def get_fr_date(self): + locale.setlocale(locale.LC_TIME, 'fr_FR') + today = datetime.datetime.now() + today = today.strftime("%d %B %Y") + locale.setlocale(locale.LC_TIME, 'C') + return today + + + async def run_daily_at_6_am(self): + while True: + now = datetime.now() + # Calculate the time until 6 AM next day + next_run = (now + timedelta(days=1)).replace(hour=6, minute=5, second=0, microsecond=0) + sleep_seconds = (next_run - now).total_seconds() + + # Wait until the next run time + await asyncio.sleep(sleep_seconds) + + # Run the daily task + await daily_task() + + + + async def daily_task(self): + + title_post = "Les news du " + self.get_fr_date() + + sub_stack_post = Post( + title=title_post, + subtitle="", + user_id=self.user_id + ) + + midnight_today = datetime.datetime.now(datetime.timezone.utc).replace(hour=0, minute=0, second=0, microsecond=0) + yesterday_6am = datetime.now(timezone.utc).replace(hour=6, minute=0, second=0, microsecond=0) - timedelta(days=1) + + formatted_date = midnight_today.strftime('%a, %d %b %Y %H:%M:%S %z') + + all_news_posts = [] + + for feed in self.feeds: + + print(feed.url) + html_text = requests.get(feed.url).text + newsFeed = feedparser.parse(html_text) + + + if feed.youtube is True: + new_posts = [entry for entry in newsFeed.entries if datetime.datetime.fromisoformat(entry.published) > yesterday_6am] + else: + new_posts = [entry for entry in newsFeed.entries if datetime.datetime.strptime(entry.published.replace('GMT', '+0000'), '%a, %d %b %Y %H:%M:%S %z') > yesterday_6am] + + all_news_posts.extend(new_posts) + + + random.shuffle(all_news_posts) + + + for post in all_news_posts: + linkURL = post["link"] + title = post["title"] + ftext = "" + + if "summary" in post: + ftext = html.unescape(post["summary"]) + # Using regular expressions to remove HTML tags + ftext = re.sub('<[^<]+?>', '', ftext) + pattern = r"L’article .* est apparu en premier sur .*" + ftext = re.sub(pattern, '', ftext) + + if "yt_videoid" in post: + sub_stack_post.add({"type":"heading", "level":3, "content": title}) + videoId = post["yt_videoid"] + print(videoId) + sub_stack_post.add({"type":"youtube2", "src": videoId }) + sub_stack_post.add({'type': 'paragraph', 'content': [ + {'content': linkURL, 'marks': [{'type': "link", 'href': linkURL}]}]}) + else: + + + + + if ftext != "": + sub_stack_post.add({"type":"heading", "level":3, "content": title}) + sub_stack_post.add({"type":"paragraph", "content": ftext }) + sub_stack_post.add({'type': 'paragraph', 'content': [ + {'content': linkURL, 'marks': [{'type': "link", 'href': linkURL}]}]}) + + if "links" in post: + for link in post["links"]: + + if link["type"] == "image/jpg": + imgUrl = link["href"] + sub_stack_post.add({'type': 'captionedImage', 'src': imgUrl}) + + + sub_stack_post.add({"type":"horizontal_rule"}) + + + + sub_stack_post.add({"type":"heading", "level":3, "content": "Sources"}) + for feed in self.feeds: + sub_stack_post.add({'type': 'paragraph', 'content': [ + {'content': feed.url, 'marks': [{'type': "link", 'href': feed.url}]}]}) + + + draft = self.api.post_draft(sub_stack_post.get_draft()) + self.api.prepublish_draft(draft.get("id")) + #self.api.publish_draft(draft.get("id")) + + + + + + + + +async def main(login, password, account): + + setuplogger() + + if os.path.exists("last_scan_date.txt"): + with open("last_scan_date.txt", "r") as f: + last_post_date = datetime.datetime.strptime(f.read().strip(), '%a, %d %b %Y %H:%M:%S %z') + else: + last_post_date = datetime.datetime.min.replace(tzinfo=datetime.timezone.utc) + + feeds = [] + + feeds.append(RSSfeed("https://www.factornews.com/rss.xml")) + feeds.append(RSSfeed("https://nofrag.com/feed")) + feeds.append(RSSfeed("https://dystopeek.fr/feed/")) + feeds.append(RSSfeed("https://thepixelpost.com/rss/")) + feeds.append(RSSfeed("https://yamukass.substack.com/feed")) + feeds.append(RSSfeed("https://www.youtube.com/feeds/videos.xml?channel_id=UC-OvBDfZGn1OdsqMBwkOI_A", True)) + + task = SubStackTask(login, password, account, feeds) + + #await task.run_daily_at_6_am() + await task.daily_task() + + +if __name__ == "__main__": + asyncio.run(main("gael.honorez@gmail.com", "f3PaTGedjFc2gkr1ypi5", "https://aggregateurjvfr.substack.com")) \ No newline at end of file From 6711db86f35706a97519e0477d632727920384e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABl=20Honorez?= Date: Sun, 31 Dec 2023 15:32:35 +0100 Subject: [PATCH 02/13] turn on automatic task --- Post_RSS_on_SubStack.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Post_RSS_on_SubStack.py b/Post_RSS_on_SubStack.py index 2a76dda..b316ea3 100644 --- a/Post_RSS_on_SubStack.py +++ b/Post_RSS_on_SubStack.py @@ -199,8 +199,8 @@ async def main(login, password, account): task = SubStackTask(login, password, account, feeds) - #await task.run_daily_at_6_am() - await task.daily_task() + await task.run_daily_at_6_am() + #await task.daily_task() if __name__ == "__main__": From 8ff4c61afa0a656a1b8107a81917225c14762b6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABl=20Honorez?= Date: Sun, 31 Dec 2023 19:23:32 +0100 Subject: [PATCH 03/13] locale to direct translation --- Post_RSS_on_SubStack.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/Post_RSS_on_SubStack.py b/Post_RSS_on_SubStack.py index b316ea3..9dd8156 100644 --- a/Post_RSS_on_SubStack.py +++ b/Post_RSS_on_SubStack.py @@ -62,12 +62,19 @@ class SubStackTask: def get_fr_date(self): - locale.setlocale(locale.LC_TIME, 'fr_FR') + # Mapping of English month names to French + months_en_to_fr = { + 'January': 'Janvier', 'February': 'Février', 'March': 'Mars', + 'April': 'Avril', 'May': 'Mai', 'June': 'Juin', + 'July': 'Juillet', 'August': 'Août', 'September': 'Septembre', + 'October': 'Octobre', 'November': 'Novembre', 'December': 'Décembre' + } today = datetime.datetime.now() - today = today.strftime("%d %B %Y") - locale.setlocale(locale.LC_TIME, 'C') - return today - + formatted_date = today.strftime("%d %B %Y") + # Replace the English month with the French month + for en, fr in months_en_to_fr.items(): + formatted_date = formatted_date.replace(en, fr) + return formatted_date async def run_daily_at_6_am(self): while True: From c5aff4b47c46027f27e3eb2a460fc8ad31c9c1f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABl=20Honorez?= Date: Sun, 31 Dec 2023 19:24:57 +0100 Subject: [PATCH 04/13] removing locale --- Post_RSS_on_SubStack.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Post_RSS_on_SubStack.py b/Post_RSS_on_SubStack.py index 9dd8156..d035d19 100644 --- a/Post_RSS_on_SubStack.py +++ b/Post_RSS_on_SubStack.py @@ -9,7 +9,6 @@ import logging import os import re from logging.handlers import RotatingFileHandler -import locale import random from substack import Api @@ -18,10 +17,6 @@ from substack.post import Post LOG = logging.getLogger('bot') LOG_PATTERN = logging.Formatter('%(asctime)s:%(levelname)s: [%(filename)s] %(message)s') - -locale.setlocale(locale.LC_TIME, 'fr_FR') - - def setuplogger(): conf_filename = None From abf46eb3577f335c3b10546a28e1078e67af2f0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABl=20Honorez?= Date: Sun, 31 Dec 2023 19:27:28 +0100 Subject: [PATCH 05/13] fixing missing module in calls --- Post_RSS_on_SubStack.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Post_RSS_on_SubStack.py b/Post_RSS_on_SubStack.py index d035d19..1171d5d 100644 --- a/Post_RSS_on_SubStack.py +++ b/Post_RSS_on_SubStack.py @@ -73,9 +73,9 @@ class SubStackTask: async def run_daily_at_6_am(self): while True: - now = datetime.now() + now = datetime.datetime.now() # Calculate the time until 6 AM next day - next_run = (now + timedelta(days=1)).replace(hour=6, minute=5, second=0, microsecond=0) + next_run = (now + datetime.timedelta(days=1)).replace(hour=6, minute=5, second=0, microsecond=0) sleep_seconds = (next_run - now).total_seconds() # Wait until the next run time From bc02cc2c22e6590c71e7379ab344bd0fead7a6ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABl=20Honorez?= Date: Mon, 1 Jan 2024 10:54:45 +0100 Subject: [PATCH 06/13] fixing timedate + subscribe button --- Post_RSS_on_SubStack.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/Post_RSS_on_SubStack.py b/Post_RSS_on_SubStack.py index 1171d5d..ae34644 100644 --- a/Post_RSS_on_SubStack.py +++ b/Post_RSS_on_SubStack.py @@ -82,7 +82,7 @@ class SubStackTask: await asyncio.sleep(sleep_seconds) # Run the daily task - await daily_task() + await self.daily_task() @@ -97,7 +97,7 @@ class SubStackTask: ) midnight_today = datetime.datetime.now(datetime.timezone.utc).replace(hour=0, minute=0, second=0, microsecond=0) - yesterday_6am = datetime.now(timezone.utc).replace(hour=6, minute=0, second=0, microsecond=0) - timedelta(days=1) + yesterday_6am = datetime.datetime.now(datetime.timezone.utc).replace(hour=6, minute=0, second=0, microsecond=0) - datetime.timedelta(days=1) formatted_date = midnight_today.strftime('%a, %d %b %Y %H:%M:%S %z') @@ -169,16 +169,11 @@ class SubStackTask: {'content': feed.url, 'marks': [{'type': "link", 'href': feed.url}]}]}) + sub_stack_post.add({"type":"subscribeWidget", "message":"Abonnez-vous gratuitement pour recevoir chaque jour les news dans votre e-mail et soutenir mon travail."}) + draft = self.api.post_draft(sub_stack_post.get_draft()) self.api.prepublish_draft(draft.get("id")) - #self.api.publish_draft(draft.get("id")) - - - - - - - + self.api.publish_draft(draft.get("id")) async def main(login, password, account): From 1395940824f3b6bfae09b460d61ecb559ac0ba5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABl=20Honorez?= Date: Mon, 1 Jan 2024 17:39:38 +0100 Subject: [PATCH 07/13] adding a feed --- Post_RSS_on_SubStack.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Post_RSS_on_SubStack.py b/Post_RSS_on_SubStack.py index ae34644..bc35549 100644 --- a/Post_RSS_on_SubStack.py +++ b/Post_RSS_on_SubStack.py @@ -192,7 +192,9 @@ async def main(login, password, account): feeds.append(RSSfeed("https://dystopeek.fr/feed/")) feeds.append(RSSfeed("https://thepixelpost.com/rss/")) feeds.append(RSSfeed("https://yamukass.substack.com/feed")) + feeds.append(RSSfeed("https://tseret.com/categorie/tests/feed")) feeds.append(RSSfeed("https://www.youtube.com/feeds/videos.xml?channel_id=UC-OvBDfZGn1OdsqMBwkOI_A", True)) + task = SubStackTask(login, password, account, feeds) From f828d8f88b79a305b616a93020c7546a4655196c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABl=20Honorez?= Date: Mon, 1 Jan 2024 18:20:22 +0100 Subject: [PATCH 08/13] adding feeds --- Post_RSS_on_SubStack.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Post_RSS_on_SubStack.py b/Post_RSS_on_SubStack.py index bc35549..237abac 100644 --- a/Post_RSS_on_SubStack.py +++ b/Post_RSS_on_SubStack.py @@ -193,7 +193,11 @@ async def main(login, password, account): feeds.append(RSSfeed("https://thepixelpost.com/rss/")) feeds.append(RSSfeed("https://yamukass.substack.com/feed")) feeds.append(RSSfeed("https://tseret.com/categorie/tests/feed")) + feeds.append(RSSfeed("https://www.gamesidestory.com/feed")) + feeds.append(RSSfeed("https://www.nintendo-town.fr/feed")) feeds.append(RSSfeed("https://www.youtube.com/feeds/videos.xml?channel_id=UC-OvBDfZGn1OdsqMBwkOI_A", True)) + + task = SubStackTask(login, password, account, feeds) From 1c34ba7ced726b4bbae30d04ea49f80f964f181b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABl=20Honorez?= Date: Mon, 1 Jan 2024 18:22:27 +0100 Subject: [PATCH 09/13] adding origami --- Post_RSS_on_SubStack.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Post_RSS_on_SubStack.py b/Post_RSS_on_SubStack.py index 237abac..f2d2471 100644 --- a/Post_RSS_on_SubStack.py +++ b/Post_RSS_on_SubStack.py @@ -196,6 +196,8 @@ async def main(login, password, account): feeds.append(RSSfeed("https://www.gamesidestory.com/feed")) feeds.append(RSSfeed("https://www.nintendo-town.fr/feed")) feeds.append(RSSfeed("https://www.youtube.com/feeds/videos.xml?channel_id=UC-OvBDfZGn1OdsqMBwkOI_A", True)) + feeds.append(RSSfeed("https://www.youtube.com/feeds/videos.xml?playlist_id=PLZRiqJjIUlDTrwYs_UqEIts5fVaBpaIEz", True)) + From 0055a0f9f5d945df2f787da4bd2b1323948f8894 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABl=20Honorez?= Date: Mon, 1 Jan 2024 18:37:57 +0100 Subject: [PATCH 10/13] adding dockerfile and requirements --- Dockerfile | 5 +++++ requirements.txt | 3 +++ 2 files changed, 8 insertions(+) create mode 100644 Dockerfile create mode 100644 requirements.txt diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..d29a013 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,5 @@ +FROM python:3.8 +WORKDIR /app +COPY . /app +RUN pip install -r requirements.txt +CMD ["python", "./Post_RSS_on_SubStack.py"] \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..7189d4d --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +requests +feedparser +python-substack \ No newline at end of file From fc134bacb0381a095be95e30dd87954879c18a84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABl=20Honorez?= Date: Mon, 1 Jan 2024 18:39:55 +0100 Subject: [PATCH 11/13] pulling from git --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d29a013..ca3c0b1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,6 @@ FROM python:3.8 +RUN apt-get update && apt-get install -y git +RUN git clone http://192.168.1.25:8124/zep/Substack_JV.git /app WORKDIR /app -COPY . /app RUN pip install -r requirements.txt CMD ["python", "./Post_RSS_on_SubStack.py"] \ No newline at end of file From be8d49415f2867f77a71208db89d70ab557fbeae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABl=20Honorez?= Date: Mon, 1 Jan 2024 19:10:05 +0100 Subject: [PATCH 12/13] adding logs --- Post_RSS_on_SubStack.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/Post_RSS_on_SubStack.py b/Post_RSS_on_SubStack.py index f2d2471..a60859c 100644 --- a/Post_RSS_on_SubStack.py +++ b/Post_RSS_on_SubStack.py @@ -77,7 +77,7 @@ class SubStackTask: # Calculate the time until 6 AM next day next_run = (now + datetime.timedelta(days=1)).replace(hour=6, minute=5, second=0, microsecond=0) sleep_seconds = (next_run - now).total_seconds() - + LOG.info("Waiting for " + str(sleep_seconds) + " seconds for next scan") # Wait until the next run time await asyncio.sleep(sleep_seconds) @@ -105,7 +105,6 @@ class SubStackTask: for feed in self.feeds: - print(feed.url) html_text = requests.get(feed.url).text newsFeed = feedparser.parse(html_text) @@ -126,6 +125,8 @@ class SubStackTask: title = post["title"] ftext = "" + LOG.info("Posting " + str(title)) + if "summary" in post: ftext = html.unescape(post["summary"]) # Using regular expressions to remove HTML tags @@ -136,7 +137,6 @@ class SubStackTask: if "yt_videoid" in post: sub_stack_post.add({"type":"heading", "level":3, "content": title}) videoId = post["yt_videoid"] - print(videoId) sub_stack_post.add({"type":"youtube2", "src": videoId }) sub_stack_post.add({'type': 'paragraph', 'content': [ {'content': linkURL, 'marks': [{'type': "link", 'href': linkURL}]}]}) @@ -197,13 +197,10 @@ async def main(login, password, account): feeds.append(RSSfeed("https://www.nintendo-town.fr/feed")) feeds.append(RSSfeed("https://www.youtube.com/feeds/videos.xml?channel_id=UC-OvBDfZGn1OdsqMBwkOI_A", True)) feeds.append(RSSfeed("https://www.youtube.com/feeds/videos.xml?playlist_id=PLZRiqJjIUlDTrwYs_UqEIts5fVaBpaIEz", True)) - - - - task = SubStackTask(login, password, account, feeds) + LOG.info("Starting bot") await task.run_daily_at_6_am() #await task.daily_task() From 39bd8ef0125b78b05a52ff71ba0d4627fc6358e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABl=20Honorez?= Date: Mon, 1 Jan 2024 19:28:54 +0100 Subject: [PATCH 13/13] change the way we get the image --- Dockerfile | 4 +++- update_and_run.sh | 7 +++++++ 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 update_and_run.sh diff --git a/Dockerfile b/Dockerfile index ca3c0b1..7f02bcc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,4 +3,6 @@ RUN apt-get update && apt-get install -y git RUN git clone http://192.168.1.25:8124/zep/Substack_JV.git /app WORKDIR /app RUN pip install -r requirements.txt -CMD ["python", "./Post_RSS_on_SubStack.py"] \ No newline at end of file +COPY update_and_run.sh /app +RUN chmod +x /app/update_and_run.sh +CMD ["./update_and_run.sh"] \ No newline at end of file diff --git a/update_and_run.sh b/update_and_run.sh new file mode 100644 index 0000000..0cd8edd --- /dev/null +++ b/update_and_run.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# Pull the latest changes +git pull origin main + +# Run your Python script +python Post_RSS_on_SubStack.py \ No newline at end of file