Compare commits
1
Commits
main
..
30f88b1d51
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
30f88b1d51 |
+4
-15
@@ -1,19 +1,8 @@
|
|||||||
FROM python:3.12
|
FROM python:3.8
|
||||||
|
|
||||||
RUN apt-get update && apt-get install -y git
|
RUN apt-get update && apt-get install -y git
|
||||||
RUN git clone https://gitea.zep.best/zep/Substack_JV.git /app
|
RUN git clone http://192.168.1.25:8124/zep/Substack_JV.git /app
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
RUN pip install --upgrade pip
|
|
||||||
COPY requirements.txt .
|
|
||||||
RUN pip install -r requirements.txt
|
RUN pip install -r requirements.txt
|
||||||
|
|
||||||
ENV TZ=Europe/Brussels
|
|
||||||
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
|
|
||||||
RUN playwright install --with-deps chromium
|
|
||||||
COPY update_and_run.sh /app
|
COPY update_and_run.sh /app
|
||||||
# Normalize line endings (Windows CRLF -> LF) and ensure readable
|
RUN chmod +x /app/update_and_run.sh
|
||||||
RUN sed -i 's/\r$//' /app/update_and_run.sh && chmod a+r /app/update_and_run.sh
|
CMD ["./update_and_run.sh"]
|
||||||
|
|
||||||
# Single entrypoint: run via sh (no exec bit required, survives noexec mounts)
|
|
||||||
ENTRYPOINT ["sh", "/app/update_and_run.sh"]
|
|
||||||
@@ -0,0 +1,209 @@
|
|||||||
|
import asyncio
|
||||||
|
import argparse
|
||||||
|
import requests
|
||||||
|
import feedparser
|
||||||
|
import io
|
||||||
|
import html
|
||||||
|
import datetime
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
from logging.handlers import RotatingFileHandler
|
||||||
|
import random
|
||||||
|
|
||||||
|
from substack import Api
|
||||||
|
from substack.post import Post
|
||||||
|
|
||||||
|
LOG = logging.getLogger('bot')
|
||||||
|
LOG_PATTERN = logging.Formatter('%(asctime)s:%(levelname)s: [%(filename)s] %(message)s')
|
||||||
|
|
||||||
|
def setuplogger():
|
||||||
|
|
||||||
|
conf_filename = None
|
||||||
|
|
||||||
|
steam_handler = logging.StreamHandler()
|
||||||
|
steam_handler.setFormatter(LOG_PATTERN)
|
||||||
|
steam_handler.setLevel(logging.DEBUG)
|
||||||
|
|
||||||
|
def setup_logger(logger_name, file_name=None, add_steam=False):
|
||||||
|
file_name = file_name or logger_name
|
||||||
|
log_filename = f"{file_name}.log"
|
||||||
|
|
||||||
|
logger = logging.getLogger(logger_name)
|
||||||
|
logger.setLevel(logging.DEBUG)
|
||||||
|
file_handler = RotatingFileHandler(log_filename, "a", 1000000, 1)
|
||||||
|
file_handler.setFormatter(LOG_PATTERN)
|
||||||
|
logger.addHandler(file_handler)
|
||||||
|
if add_steam:
|
||||||
|
logger.addHandler(steam_handler)
|
||||||
|
|
||||||
|
setup_logger("bot", conf_filename, True)
|
||||||
|
|
||||||
|
class RSSfeed():
|
||||||
|
def __init__(self, url, yt=False):
|
||||||
|
self.url = url
|
||||||
|
self.youtube = yt
|
||||||
|
|
||||||
|
class SubStackTask:
|
||||||
|
def __init__(self, login, password, account, feeds):
|
||||||
|
self.api = Api(
|
||||||
|
email=login,
|
||||||
|
password=password,
|
||||||
|
publication_url=account,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.user_id = self.api.get_user_id()
|
||||||
|
self.feeds = feeds
|
||||||
|
|
||||||
|
|
||||||
|
def get_fr_date(self):
|
||||||
|
# Mapping of English month names to French
|
||||||
|
months_en_to_fr = {
|
||||||
|
'January': 'Janvier', 'February': 'Février', 'March': 'Mars',
|
||||||
|
'April': 'Avril', 'May': 'Mai', 'June': 'Juin',
|
||||||
|
'July': 'Juillet', 'August': 'Août', 'September': 'Septembre',
|
||||||
|
'October': 'Octobre', 'November': 'Novembre', 'December': 'Décembre'
|
||||||
|
}
|
||||||
|
today = datetime.datetime.now()
|
||||||
|
formatted_date = today.strftime("%d %B %Y")
|
||||||
|
# Replace the English month with the French month
|
||||||
|
for en, fr in months_en_to_fr.items():
|
||||||
|
formatted_date = formatted_date.replace(en, fr)
|
||||||
|
return formatted_date
|
||||||
|
|
||||||
|
async def run_daily_at_6_am(self):
|
||||||
|
while True:
|
||||||
|
now = datetime.datetime.now()
|
||||||
|
# Calculate the time until 6 AM next day
|
||||||
|
next_run = (now + datetime.timedelta(days=1)).replace(hour=6, minute=5, second=0, microsecond=0)
|
||||||
|
sleep_seconds = (next_run - now).total_seconds()
|
||||||
|
LOG.info("Waiting for " + str(sleep_seconds) + " seconds for next scan")
|
||||||
|
# Wait until the next run time
|
||||||
|
await asyncio.sleep(sleep_seconds)
|
||||||
|
|
||||||
|
# Run the daily task
|
||||||
|
await self.daily_task()
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
async def daily_task(self):
|
||||||
|
|
||||||
|
title_post = "Les news du " + self.get_fr_date()
|
||||||
|
|
||||||
|
sub_stack_post = Post(
|
||||||
|
title=title_post,
|
||||||
|
subtitle="",
|
||||||
|
user_id=self.user_id
|
||||||
|
)
|
||||||
|
|
||||||
|
midnight_today = datetime.datetime.now(datetime.timezone.utc).replace(hour=0, minute=0, second=0, microsecond=0)
|
||||||
|
yesterday_6am = datetime.datetime.now(datetime.timezone.utc).replace(hour=6, minute=0, second=0, microsecond=0) - datetime.timedelta(days=1)
|
||||||
|
|
||||||
|
formatted_date = midnight_today.strftime('%a, %d %b %Y %H:%M:%S %z')
|
||||||
|
|
||||||
|
all_news_posts = []
|
||||||
|
|
||||||
|
for feed in self.feeds:
|
||||||
|
|
||||||
|
html_text = requests.get(feed.url).text
|
||||||
|
newsFeed = feedparser.parse(html_text)
|
||||||
|
|
||||||
|
|
||||||
|
if feed.youtube is True:
|
||||||
|
new_posts = [entry for entry in newsFeed.entries if datetime.datetime.fromisoformat(entry.published) > yesterday_6am]
|
||||||
|
else:
|
||||||
|
new_posts = [entry for entry in newsFeed.entries if datetime.datetime.strptime(entry.published.replace('GMT', '+0000'), '%a, %d %b %Y %H:%M:%S %z') > yesterday_6am]
|
||||||
|
|
||||||
|
all_news_posts.extend(new_posts)
|
||||||
|
|
||||||
|
|
||||||
|
random.shuffle(all_news_posts)
|
||||||
|
|
||||||
|
|
||||||
|
for post in all_news_posts:
|
||||||
|
linkURL = post["link"]
|
||||||
|
title = post["title"]
|
||||||
|
ftext = ""
|
||||||
|
|
||||||
|
LOG.info("Posting " + str(title))
|
||||||
|
|
||||||
|
if "summary" in post:
|
||||||
|
ftext = html.unescape(post["summary"])
|
||||||
|
# Using regular expressions to remove HTML tags
|
||||||
|
ftext = re.sub('<[^<]+?>', '', ftext)
|
||||||
|
pattern = r"L’article .* est apparu en premier sur .*"
|
||||||
|
ftext = re.sub(pattern, '', ftext)
|
||||||
|
|
||||||
|
if "yt_videoid" in post:
|
||||||
|
sub_stack_post.add({"type":"heading", "level":3, "content": title})
|
||||||
|
videoId = post["yt_videoid"]
|
||||||
|
sub_stack_post.add({"type":"youtube2", "src": videoId })
|
||||||
|
sub_stack_post.add({'type': 'paragraph', 'content': [
|
||||||
|
{'content': linkURL, 'marks': [{'type': "link", 'href': linkURL}]}]})
|
||||||
|
else:
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
if ftext != "":
|
||||||
|
sub_stack_post.add({"type":"heading", "level":3, "content": title})
|
||||||
|
sub_stack_post.add({"type":"paragraph", "content": ftext })
|
||||||
|
sub_stack_post.add({'type': 'paragraph', 'content': [
|
||||||
|
{'content': linkURL, 'marks': [{'type': "link", 'href': linkURL}]}]})
|
||||||
|
|
||||||
|
if "links" in post:
|
||||||
|
for link in post["links"]:
|
||||||
|
|
||||||
|
if link["type"] == "image/jpg":
|
||||||
|
imgUrl = link["href"]
|
||||||
|
sub_stack_post.add({'type': 'captionedImage', 'src': imgUrl})
|
||||||
|
|
||||||
|
|
||||||
|
sub_stack_post.add({"type":"horizontal_rule"})
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
sub_stack_post.add({"type":"heading", "level":3, "content": "Sources"})
|
||||||
|
for feed in self.feeds:
|
||||||
|
sub_stack_post.add({'type': 'paragraph', 'content': [
|
||||||
|
{'content': feed.url, 'marks': [{'type': "link", 'href': feed.url}]}]})
|
||||||
|
|
||||||
|
|
||||||
|
sub_stack_post.add({"type":"subscribeWidget", "message":"Abonnez-vous gratuitement pour recevoir chaque jour les news dans votre e-mail et soutenir mon travail."})
|
||||||
|
|
||||||
|
draft = self.api.post_draft(sub_stack_post.get_draft())
|
||||||
|
self.api.prepublish_draft(draft.get("id"))
|
||||||
|
self.api.publish_draft(draft.get("id"))
|
||||||
|
|
||||||
|
async def main(login, password, account):
|
||||||
|
|
||||||
|
setuplogger()
|
||||||
|
|
||||||
|
if os.path.exists("last_scan_date.txt"):
|
||||||
|
with open("last_scan_date.txt", "r") as f:
|
||||||
|
last_post_date = datetime.datetime.strptime(f.read().strip(), '%a, %d %b %Y %H:%M:%S %z')
|
||||||
|
else:
|
||||||
|
last_post_date = datetime.datetime.min.replace(tzinfo=datetime.timezone.utc)
|
||||||
|
|
||||||
|
feeds = []
|
||||||
|
|
||||||
|
feeds.append(RSSfeed("https://www.factornews.com/rss.xml"))
|
||||||
|
feeds.append(RSSfeed("https://nofrag.com/feed"))
|
||||||
|
feeds.append(RSSfeed("https://dystopeek.fr/feed/"))
|
||||||
|
feeds.append(RSSfeed("https://thepixelpost.com/rss/"))
|
||||||
|
feeds.append(RSSfeed("https://yamukass.substack.com/feed"))
|
||||||
|
feeds.append(RSSfeed("https://tseret.com/categorie/tests/feed"))
|
||||||
|
feeds.append(RSSfeed("https://www.gamesidestory.com/feed"))
|
||||||
|
feeds.append(RSSfeed("https://www.nintendo-town.fr/feed"))
|
||||||
|
feeds.append(RSSfeed("https://www.youtube.com/feeds/videos.xml?channel_id=UC-OvBDfZGn1OdsqMBwkOI_A", True))
|
||||||
|
feeds.append(RSSfeed("https://www.youtube.com/feeds/videos.xml?playlist_id=PLZRiqJjIUlDTrwYs_UqEIts5fVaBpaIEz", True))
|
||||||
|
|
||||||
|
task = SubStackTask(login, password, account, feeds)
|
||||||
|
|
||||||
|
LOG.info("Starting bot")
|
||||||
|
await task.run_daily_at_6_am()
|
||||||
|
#await task.daily_task()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
asyncio.run(main("gael.honorez@gmail.com", "f3PaTGedjFc2gkr1ypi5", "https://aggregateurjvfr.substack.com"))
|
||||||
-162
@@ -1,162 +0,0 @@
|
|||||||
# backfill_from_ghost.py
|
|
||||||
from __future__ import annotations
|
|
||||||
import os, re, sys, html
|
|
||||||
from typing import Dict, List, Optional
|
|
||||||
import requests
|
|
||||||
from bs4 import BeautifulSoup
|
|
||||||
|
|
||||||
# Reuse your existing GhostAdmin client (same headers/base/proxy behavior)
|
|
||||||
# Adjust the import path if your Ghost client lives elsewhere.
|
|
||||||
from presquegratos import GhostAdmin
|
|
||||||
|
|
||||||
from storage import Storage
|
|
||||||
from keys import xgp_key, egs_key, psplus_key
|
|
||||||
|
|
||||||
# ---------------- Ghost helpers (reusing your admin client) ----------------
|
|
||||||
def ghost_list_posts(ghost: GhostAdmin, page: int = 1) -> Dict:
|
|
||||||
# Minimal params: avoid 'filter' and 'fields' to dodge 400 behind __bot proxy
|
|
||||||
url = ghost.base + "posts/"
|
|
||||||
params = {
|
|
||||||
"limit": "50",
|
|
||||||
"page": str(page),
|
|
||||||
"order": "published_at DESC",
|
|
||||||
"formats": "lexical,html", # <-- IMPORTANT
|
|
||||||
}
|
|
||||||
r = requests.get(url, headers=ghost._headers(), params=params, timeout=30)
|
|
||||||
r.raise_for_status()
|
|
||||||
return r.json()
|
|
||||||
|
|
||||||
def list_recap_posts(ghost: GhostAdmin, hard_limit: int = 2000) -> List[Dict]:
|
|
||||||
posts: List[Dict] = []
|
|
||||||
page = 1
|
|
||||||
while True:
|
|
||||||
data = ghost_list_posts(ghost, page=page)
|
|
||||||
batch = data.get("posts", [])
|
|
||||||
if not batch:
|
|
||||||
break
|
|
||||||
# client-side filter to be robust to proxy quirks
|
|
||||||
for p in batch:
|
|
||||||
title = (p.get("title") or "").strip()
|
|
||||||
if title.startswith("Récap hebdo"):
|
|
||||||
posts.append(p)
|
|
||||||
if len(batch) < 50 or len(posts) >= hard_limit:
|
|
||||||
break
|
|
||||||
page += 1
|
|
||||||
return posts
|
|
||||||
|
|
||||||
# ---------------- Parsing helpers (unchanged) ----------------
|
|
||||||
#MS_STORE_RE = re.compile(r"(?:microsoft|xbox)\.com/.*/store/.*/([0-9A-Z]{12,})", re.I)
|
|
||||||
MS_STORE_RE = re.compile(r"(?:xbox|microsoft)\.com/.*/store/.*/([0-9A-Z]{12,16})", re.I)
|
|
||||||
EPIC_RE = re.compile(r"epicgames\.com/store/.*/p/([\w\-]+)", re.I)
|
|
||||||
PSBLOG_RE = re.compile(r"blog\.playstation\.com/.*", re.I)
|
|
||||||
|
|
||||||
def clean_text(s: str) -> str:
|
|
||||||
return re.sub(r"\s+", " ", html.unescape(s or "")).strip()
|
|
||||||
|
|
||||||
def extract_sections(soup: BeautifulSoup) -> Dict[str, BeautifulSoup]:
|
|
||||||
sections: Dict[str, BeautifulSoup] = {}
|
|
||||||
current = None
|
|
||||||
current_key = None
|
|
||||||
for node in soup.find_all(["h2","h3","h4","p","ul","ol","div","section"]):
|
|
||||||
if node.name in ("h2","h3","h4"):
|
|
||||||
title = clean_text(node.get_text())
|
|
||||||
key = None
|
|
||||||
tl = title.lower()
|
|
||||||
if "game pass" in tl:
|
|
||||||
key = "xgp"
|
|
||||||
elif "egs" in tl or "epic" in tl:
|
|
||||||
key = "egs"
|
|
||||||
elif "ps plus" in tl or "ps+" in tl:
|
|
||||||
key = "psplus"
|
|
||||||
if key:
|
|
||||||
current_key = key
|
|
||||||
current = sections[key] = soup.new_tag("div")
|
|
||||||
continue
|
|
||||||
if current_key and current is not None:
|
|
||||||
current.append(node)
|
|
||||||
return sections
|
|
||||||
|
|
||||||
def parse_xgp(section: BeautifulSoup) -> List[Dict]:
|
|
||||||
items = []
|
|
||||||
for a in section.find_all("a", href=True):
|
|
||||||
href = a["href"]
|
|
||||||
m = MS_STORE_RE.search(href)
|
|
||||||
title = clean_text(a.get_text())
|
|
||||||
if m or title:
|
|
||||||
productId = m.group(1) if m else None
|
|
||||||
items.append({"title": title, "productId": productId})
|
|
||||||
uniq, seen = [], set()
|
|
||||||
for it in items:
|
|
||||||
k = xgp_key(it)
|
|
||||||
if k not in seen:
|
|
||||||
uniq.append(it); seen.add(k)
|
|
||||||
return uniq
|
|
||||||
|
|
||||||
def parse_egs(section: BeautifulSoup) -> List[Dict]:
|
|
||||||
items = []
|
|
||||||
for a in section.find_all("a", href=True):
|
|
||||||
if not EPIC_RE.search(a["href"]):
|
|
||||||
continue
|
|
||||||
title = clean_text(a.get_text()) or clean_text(a.get("title"))
|
|
||||||
items.append({"title": title, "start": ""})
|
|
||||||
uniq, seen = [], set()
|
|
||||||
for it in items:
|
|
||||||
k = egs_key(it)
|
|
||||||
if k not in seen:
|
|
||||||
uniq.append(it); seen.add(k)
|
|
||||||
return uniq
|
|
||||||
|
|
||||||
def parse_psplus(section: BeautifulSoup, post_title: str) -> Optional[Dict]:
|
|
||||||
a = section.find("a", href=PSBLOG_RE)
|
|
||||||
url = a["href"] if a else ""
|
|
||||||
m = re.search(r"(\d{2})-(\d{2})-(\d{4})", post_title)
|
|
||||||
iso = ""
|
|
||||||
if m:
|
|
||||||
d, mth, y = m.group(1), m.group(2), m.group(3)
|
|
||||||
iso = f"{y}-{mth}-{d}"
|
|
||||||
return {"url": url, "date": iso}
|
|
||||||
|
|
||||||
# ---------------- Main backfill ----------------
|
|
||||||
def backfill():
|
|
||||||
# Use the same env your main script uses; GhostAdmin will read them internally or
|
|
||||||
# you can pass them explicitly if your class expects (base_url, admin_key).
|
|
||||||
ghost = GhostAdmin(
|
|
||||||
admin_url=os.environ.get("GHOST_ADMIN_URL", "").rstrip("/") + "/",
|
|
||||||
admin_key=os.environ.get("GHOST_ADMIN_KEY", "")
|
|
||||||
)
|
|
||||||
store = Storage()
|
|
||||||
|
|
||||||
posts = list_recap_posts(ghost)
|
|
||||||
print(f"Found {len(posts)} recap posts.")
|
|
||||||
|
|
||||||
total_xgp = total_egs = total_ps = 0
|
|
||||||
|
|
||||||
dedup = []
|
|
||||||
|
|
||||||
for p in posts:
|
|
||||||
|
|
||||||
pid = p["id"]
|
|
||||||
title = p.get("title") or ""
|
|
||||||
html_body = p.get("html") or ""
|
|
||||||
|
|
||||||
soup = BeautifulSoup(html_body, "html.parser")
|
|
||||||
sections = extract_sections(soup)
|
|
||||||
|
|
||||||
for it in parse_xgp(sections.get("xgp", BeautifulSoup("", "html.parser"))):
|
|
||||||
key = xgp_key(it)
|
|
||||||
if not key in dedup:
|
|
||||||
store.remember("xgp", key, pid); total_xgp += 1
|
|
||||||
dedup.append(key)
|
|
||||||
|
|
||||||
# for it in parse_egs(sections.get("egs", BeautifulSoup("", "html.parser"))):
|
|
||||||
# store.remember("egs", egs_key(it), pid); total_egs += 1
|
|
||||||
# if "psplus" in sections:
|
|
||||||
# item = parse_psplus(sections["psplus"], title)
|
|
||||||
# store.remember("psplus", psplus_key(item), pid); total_ps += 1
|
|
||||||
|
|
||||||
print(f"Backfilled from: {title}")
|
|
||||||
|
|
||||||
print(f"Done. Inserted ~ XGP:{total_xgp} | EGS:{total_egs} | PS+:{total_ps}")
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
backfill()
|
|
||||||
@@ -1 +0,0 @@
|
|||||||
{"__cf_bm": "95up0icsYyESvD6suTUFG05xaWxwEr5_xuHUOv32G9I-1720025055-1.0.1.1-NlvsLW9j26FX8aPpLmVETEJ0zd.VyXefLr75kvT6iC.zHnPtkbIWgfesI0VaUGuvwV62qHpctJEoahLR9TIuHQ", "ab_experiment_sampled": "%22false%22", "ab_testing_id": "%22a6e7ba67-7dc0-452c-a935-d2f2bddd5edf%22", "ajs_anonymous_id": "%22e4535e95-1c5b-4173-82db-47807c57fb38%22", "cookie_storage_key": "f666a42c-49e8-47a2-bdbc-6eece0d6a06e", "substack.sid": "s%3ARLYSI2_XaTlGuYIpTYWjS8ib48PpuE0S.jNwCzcGzKUvUAuFdLNdfgxwewTUawIoDDZ05moubvzM", "visit_id": "%7B%22id%22%3A%22a0d46be8-56f4-406f-b1d7-14c41369b737%22%2C%22timestamp%22%3A%222024-07-03T16%3A44%3A13.349Z%22%7D", "AWSALBTG": "yw2xMbYVFbKWSzJiQsdCKp7mMH+wQ5T4/JIUc1TvywUi5iIJVXuO21AMhb+oPgegicdtpekLTDTl+zWKEekRsurS7+20skhmPxZXJf/Tl7jBd/PecbW7qa3DHkPvQtWz+SWD8+7P1rNjmY9lmyZgzH/ZeGgeiishRz9gsGO0OT/d", "AWSALBTGCORS": "yw2xMbYVFbKWSzJiQsdCKp7mMH+wQ5T4/JIUc1TvywUi5iIJVXuO21AMhb+oPgegicdtpekLTDTl+zWKEekRsurS7+20skhmPxZXJf/Tl7jBd/PecbW7qa3DHkPvQtWz+SWD8+7P1rNjmY9lmyZgzH/ZeGgeiishRz9gsGO0OT/d"}
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
version: '3.3'
|
|
||||||
|
|
||||||
services:
|
|
||||||
substackjv:
|
|
||||||
build: .
|
|
||||||
volumes:
|
|
||||||
- /path/to/your/host/directory:/data
|
|
||||||
environment:
|
|
||||||
- TZ=Europe/Brussels
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
set GHOST_ADMIN_KEY=68bad0e13546e700012dd65d:116a81b7e189d3b3d3b86082f97ef65daedb06498a3f1f902b8e0c08d095dc19
|
|
||||||
set GHOST_ADMIN_URL=https://ghostadmin.zep.best/ghost/api/admin/__bot/FF4440EBA737506D397C170A8422109C357AA7582F10938B7C5F11D6B652F5D4
|
|
||||||
set GHOST_EMAIL_SEGMENT=status:free
|
|
||||||
set GHOST_NEWSLETTER_SLUG=default-newsletter
|
|
||||||
set GHOST_CONTENT_URL=https://ghost.zep.best
|
|
||||||
set DB_FILE_FALLBACK=f:\workspace\Substack_JV\data\published.db
|
|
||||||
set MISTRAL_API_KEY=tQJHvYlmwz1ihKxOhXS3FmDNTRhBh6b3
|
|
||||||
@@ -1,15 +0,0 @@
|
|||||||
https://www.factornews.com/rss.xml
|
|
||||||
https://nofrag.com/feed
|
|
||||||
https://dystopeek.fr/feed/
|
|
||||||
https://thepixelpost.com/rss/
|
|
||||||
https://yamukass.substack.com/feed
|
|
||||||
https://tseret.com/categorie/tests/feed
|
|
||||||
https://www.gamesidestory.com/feed
|
|
||||||
https://www.nintendo-town.fr/feed
|
|
||||||
https://jesuisungameur.com/feed
|
|
||||||
https://www.switch-actu.fr/categorie/tests/tests-de-jeux/feed
|
|
||||||
https://www.playscope.com/category/articles/test-gaming/feed
|
|
||||||
https://jrpgfr.net/category/test/feed
|
|
||||||
https://jv.jeuxonline.info/rss/dossiers/rss.xml
|
|
||||||
https://www.youtube.com/feeds/videos.xml?channel_id=UC-OvBDfZGn1OdsqMBwkOI_A
|
|
||||||
https://www.youtube.com/feeds/videos.xml?playlist_id=PLZRiqJjIUlDTrwYs_UqEIts5fVaBpaIEz
|
|
||||||
@@ -1,20 +0,0 @@
|
|||||||
# keys.py (or inline in your main)
|
|
||||||
def xgp_key(item) -> str:
|
|
||||||
# Prefer stable Microsoft Store productId if present; fallback to normalized title.
|
|
||||||
pid = (item.get("productId") or "").strip()
|
|
||||||
if pid:
|
|
||||||
return f"item:xgp:{pid}"
|
|
||||||
title = (item.get("title") or "").strip().lower()
|
|
||||||
return f"item:xgp:title:{title}"
|
|
||||||
|
|
||||||
def egs_key(item) -> str:
|
|
||||||
# Use title + start window (your fetcher usually knows the free-week start)
|
|
||||||
title = (item.get("title") or "").strip()
|
|
||||||
start = (item.get("start") or "").strip() # ISO or YYYY-MM-DD
|
|
||||||
return f"item:egs:{title}|{start}"
|
|
||||||
|
|
||||||
def psplus_key(item) -> str:
|
|
||||||
# Use official PS Blog URL + the published month (or your computed date)
|
|
||||||
url = (item.get("url") or "").strip()
|
|
||||||
date = (item.get("date") or "").strip()
|
|
||||||
return f"item:psplus:{url}|{date}"
|
|
||||||
File diff suppressed because it is too large
Load Diff
-1040
File diff suppressed because it is too large
Load Diff
+2
-6
@@ -1,7 +1,3 @@
|
|||||||
|
requests
|
||||||
feedparser
|
feedparser
|
||||||
PyJWT>=2.7,<3
|
python-substack
|
||||||
requests>=2.31
|
|
||||||
feedparser>=6.0
|
|
||||||
aiohttp
|
|
||||||
bs4
|
|
||||||
playwright
|
|
||||||
-49
@@ -1,49 +0,0 @@
|
|||||||
# storage.py
|
|
||||||
from __future__ import annotations
|
|
||||||
import sqlite3, pathlib, datetime as dt
|
|
||||||
from typing import Optional, Iterable, Tuple
|
|
||||||
import os
|
|
||||||
DB_PATH = "/data/published.db" # bind-mount ./data:/data in docker
|
|
||||||
|
|
||||||
_SCHEMA = """
|
|
||||||
PRAGMA journal_mode = WAL;
|
|
||||||
CREATE TABLE IF NOT EXISTS published_items(
|
|
||||||
platform TEXT NOT NULL, -- e.g. xgp | egs | psplus
|
|
||||||
key TEXT PRIMARY KEY, -- your dedupe key (see below)
|
|
||||||
first_seen_utc TEXT NOT NULL, -- ISO-8601
|
|
||||||
last_post_id TEXT -- Ghost post id that recorded it
|
|
||||||
);
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_platform ON published_items(platform);
|
|
||||||
"""
|
|
||||||
|
|
||||||
class Storage:
|
|
||||||
def __init__(self, db_path: str = DB_PATH):
|
|
||||||
pathlib.Path(db_path).parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
self.conn = sqlite3.connect(db_path)
|
|
||||||
self.conn.execute("PRAGMA foreign_keys = ON;")
|
|
||||||
for stmt in filter(None, _SCHEMA.split(";")):
|
|
||||||
if stmt.strip():
|
|
||||||
self.conn.execute(stmt)
|
|
||||||
|
|
||||||
def seen(self, key: str) -> bool:
|
|
||||||
cur = self.conn.execute("SELECT 1 FROM published_items WHERE key=?", (key,))
|
|
||||||
return cur.fetchone() is not None
|
|
||||||
|
|
||||||
def remember(self, platform: str, key: str, post_id: Optional[str]):
|
|
||||||
self.conn.execute(
|
|
||||||
"INSERT OR IGNORE INTO published_items(platform,key,first_seen_utc,last_post_id) VALUES(?,?,?,?)",
|
|
||||||
(platform, key, dt.datetime.utcnow().isoformat(), post_id),
|
|
||||||
)
|
|
||||||
if post_id:
|
|
||||||
self.conn.execute("UPDATE published_items SET last_post_id=? WHERE key=?", (post_id, key))
|
|
||||||
self.conn.commit()
|
|
||||||
|
|
||||||
def bulk_remember(self, platform: str, pairs: Iterable[Tuple[str, Optional[str]]]):
|
|
||||||
rows = [(platform, k, dt.datetime.utcnow().isoformat(), pid) for (k, pid) in pairs]
|
|
||||||
self.conn.executemany(
|
|
||||||
"INSERT OR IGNORE INTO published_items(platform,key,first_seen_utc,last_post_id) VALUES(?,?,?,?)",
|
|
||||||
rows
|
|
||||||
)
|
|
||||||
self.conn.commit()
|
|
||||||
|
|
||||||
|
|
||||||
+5
-56
@@ -1,58 +1,7 @@
|
|||||||
#!/bin/sh
|
#!/bin/bash
|
||||||
set -eu
|
|
||||||
|
|
||||||
log() { printf '%s %s\n' "[$(date -u +%FT%TZ)]" "$*"; }
|
# Pull the latest changes
|
||||||
|
git pull origin main
|
||||||
|
|
||||||
stop() {
|
# Run your Python script
|
||||||
log "stopping..."
|
python Post_RSS_on_SubStack.py
|
||||||
[ -n "${PID1-}" ] && kill -TERM "$PID1" 2>/dev/null || true
|
|
||||||
[ -n "${PID2-}" ] && kill -TERM "$PID2" 2>/dev/null || true
|
|
||||||
[ -n "${TPID-}" ] && kill -TERM "$TPID" 2>/dev/null || true
|
|
||||||
wait || true
|
|
||||||
exit 0
|
|
||||||
}
|
|
||||||
trap stop INT TERM
|
|
||||||
|
|
||||||
cd /app
|
|
||||||
export GIT_TERMINAL_PROMPT=0
|
|
||||||
|
|
||||||
# MAJ forcée du code à chaque (re)démarrage
|
|
||||||
if [ -d .git ]; then
|
|
||||||
i=0
|
|
||||||
while [ $i -lt 5 ]; do
|
|
||||||
if git fetch --all --prune && git reset --hard origin/main; then
|
|
||||||
log "git updated to origin/main"
|
|
||||||
break
|
|
||||||
fi
|
|
||||||
i=$((i+1))
|
|
||||||
log "git update failed (attempt $i/5); retrying in 10s..."
|
|
||||||
sleep 10
|
|
||||||
done
|
|
||||||
[ $i -ge 5 ] && log "WARNING: git update failed after 5 attempts — continuing with current code"
|
|
||||||
else
|
|
||||||
log "WARNING: /app is not a git repo; skipping git update"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Dossiers logs
|
|
||||||
mkdir -p /var/log
|
|
||||||
: > /var/log/daily.log
|
|
||||||
: > /var/log/weekly.log
|
|
||||||
|
|
||||||
# Lancer les 2 bots (logs non bufferisés)
|
|
||||||
python -u post_rss_to_ghost.py > /var/log/daily.log 2>&1 & PID1=$!
|
|
||||||
python -u presquegratos.py > /var/log/weekly.log 2>&1 & PID2=$!
|
|
||||||
|
|
||||||
# Suivre les 2 fichiers de logs dans la sortie du conteneur
|
|
||||||
tail -F /var/log/daily.log /var/log/weekly.log &
|
|
||||||
TPID=$!
|
|
||||||
|
|
||||||
# Attente portable (pas de wait -n en /bin/sh)
|
|
||||||
while :; do
|
|
||||||
if ! kill -0 "$PID1" 2>/dev/null; then wait "$PID1" || true; break; fi
|
|
||||||
if ! kill -0 "$PID2" 2>/dev/null; then wait "$PID2" || true; break; fi
|
|
||||||
sleep 1
|
|
||||||
done
|
|
||||||
|
|
||||||
# Si un des scripts sort, on arrête le tail (le trap TERM arrêtera l'autre script)
|
|
||||||
kill -TERM "$TPID" 2>/dev/null || true
|
|
||||||
wait || true
|
|
||||||
-23
@@ -1,23 +0,0 @@
|
|||||||
import feedparser
|
|
||||||
import io
|
|
||||||
import html
|
|
||||||
import datetime
|
|
||||||
import requests
|
|
||||||
import time
|
|
||||||
|
|
||||||
url = r'https://www.xboxygen.com/spip.php?page=backend'
|
|
||||||
|
|
||||||
html_text = requests.get(url).text
|
|
||||||
news = feedparser.parse(html_text)
|
|
||||||
|
|
||||||
yesterday_6am = datetime.datetime.now(datetime.timezone.utc).replace(hour=6, minute=0, second=0, microsecond=0) - datetime.timedelta(days=1)
|
|
||||||
|
|
||||||
try:
|
|
||||||
new_posts = [entry for entry in news.entries if datetime.datetime.strptime(entry.published.replace('GMT', '+0000'), '%a, %d %b %Y %H:%M:%S %z') > yesterday_6am]
|
|
||||||
|
|
||||||
except:
|
|
||||||
new_posts = [entry for entry in news.entries if datetime.datetime.fromtimestamp(time.mktime(entry.updated_parsed)).replace(tzinfo=datetime.timezone.utc) > yesterday_6am]
|
|
||||||
#else if
|
|
||||||
#entry.updated.replace('GMT', '+0000'), '%a, %d %b %Y %H:%M:%S %z'
|
|
||||||
|
|
||||||
print(new_posts)
|
|
||||||
Reference in New Issue
Block a user