Compare commits
1
Commits
main
..
30f88b1d51
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
30f88b1d51 |
+4
-15
@@ -1,19 +1,8 @@
|
||||
FROM python:3.12
|
||||
|
||||
FROM python:3.8
|
||||
RUN apt-get update && apt-get install -y git
|
||||
RUN git clone https://gitea.zep.best/zep/Substack_JV.git /app
|
||||
RUN git clone http://192.168.1.25:8124/zep/Substack_JV.git /app
|
||||
WORKDIR /app
|
||||
|
||||
RUN pip install --upgrade pip
|
||||
COPY requirements.txt .
|
||||
RUN pip install -r requirements.txt
|
||||
|
||||
ENV TZ=Europe/Brussels
|
||||
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
|
||||
RUN playwright install --with-deps chromium
|
||||
COPY update_and_run.sh /app
|
||||
# Normalize line endings (Windows CRLF -> LF) and ensure readable
|
||||
RUN sed -i 's/\r$//' /app/update_and_run.sh && chmod a+r /app/update_and_run.sh
|
||||
|
||||
# Single entrypoint: run via sh (no exec bit required, survives noexec mounts)
|
||||
ENTRYPOINT ["sh", "/app/update_and_run.sh"]
|
||||
RUN chmod +x /app/update_and_run.sh
|
||||
CMD ["./update_and_run.sh"]
|
||||
@@ -0,0 +1,209 @@
|
||||
import asyncio
|
||||
import argparse
|
||||
import requests
|
||||
import feedparser
|
||||
import io
|
||||
import html
|
||||
import datetime
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from logging.handlers import RotatingFileHandler
|
||||
import random
|
||||
|
||||
from substack import Api
|
||||
from substack.post import Post
|
||||
|
||||
LOG = logging.getLogger('bot')
|
||||
LOG_PATTERN = logging.Formatter('%(asctime)s:%(levelname)s: [%(filename)s] %(message)s')
|
||||
|
||||
def setuplogger():
|
||||
|
||||
conf_filename = None
|
||||
|
||||
steam_handler = logging.StreamHandler()
|
||||
steam_handler.setFormatter(LOG_PATTERN)
|
||||
steam_handler.setLevel(logging.DEBUG)
|
||||
|
||||
def setup_logger(logger_name, file_name=None, add_steam=False):
|
||||
file_name = file_name or logger_name
|
||||
log_filename = f"{file_name}.log"
|
||||
|
||||
logger = logging.getLogger(logger_name)
|
||||
logger.setLevel(logging.DEBUG)
|
||||
file_handler = RotatingFileHandler(log_filename, "a", 1000000, 1)
|
||||
file_handler.setFormatter(LOG_PATTERN)
|
||||
logger.addHandler(file_handler)
|
||||
if add_steam:
|
||||
logger.addHandler(steam_handler)
|
||||
|
||||
setup_logger("bot", conf_filename, True)
|
||||
|
||||
class RSSfeed():
|
||||
def __init__(self, url, yt=False):
|
||||
self.url = url
|
||||
self.youtube = yt
|
||||
|
||||
class SubStackTask:
|
||||
def __init__(self, login, password, account, feeds):
|
||||
self.api = Api(
|
||||
email=login,
|
||||
password=password,
|
||||
publication_url=account,
|
||||
)
|
||||
|
||||
self.user_id = self.api.get_user_id()
|
||||
self.feeds = feeds
|
||||
|
||||
|
||||
def get_fr_date(self):
|
||||
# Mapping of English month names to French
|
||||
months_en_to_fr = {
|
||||
'January': 'Janvier', 'February': 'Février', 'March': 'Mars',
|
||||
'April': 'Avril', 'May': 'Mai', 'June': 'Juin',
|
||||
'July': 'Juillet', 'August': 'Août', 'September': 'Septembre',
|
||||
'October': 'Octobre', 'November': 'Novembre', 'December': 'Décembre'
|
||||
}
|
||||
today = datetime.datetime.now()
|
||||
formatted_date = today.strftime("%d %B %Y")
|
||||
# Replace the English month with the French month
|
||||
for en, fr in months_en_to_fr.items():
|
||||
formatted_date = formatted_date.replace(en, fr)
|
||||
return formatted_date
|
||||
|
||||
async def run_daily_at_6_am(self):
|
||||
while True:
|
||||
now = datetime.datetime.now()
|
||||
# Calculate the time until 6 AM next day
|
||||
next_run = (now + datetime.timedelta(days=1)).replace(hour=6, minute=5, second=0, microsecond=0)
|
||||
sleep_seconds = (next_run - now).total_seconds()
|
||||
LOG.info("Waiting for " + str(sleep_seconds) + " seconds for next scan")
|
||||
# Wait until the next run time
|
||||
await asyncio.sleep(sleep_seconds)
|
||||
|
||||
# Run the daily task
|
||||
await self.daily_task()
|
||||
|
||||
|
||||
|
||||
async def daily_task(self):
|
||||
|
||||
title_post = "Les news du " + self.get_fr_date()
|
||||
|
||||
sub_stack_post = Post(
|
||||
title=title_post,
|
||||
subtitle="",
|
||||
user_id=self.user_id
|
||||
)
|
||||
|
||||
midnight_today = datetime.datetime.now(datetime.timezone.utc).replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
yesterday_6am = datetime.datetime.now(datetime.timezone.utc).replace(hour=6, minute=0, second=0, microsecond=0) - datetime.timedelta(days=1)
|
||||
|
||||
formatted_date = midnight_today.strftime('%a, %d %b %Y %H:%M:%S %z')
|
||||
|
||||
all_news_posts = []
|
||||
|
||||
for feed in self.feeds:
|
||||
|
||||
html_text = requests.get(feed.url).text
|
||||
newsFeed = feedparser.parse(html_text)
|
||||
|
||||
|
||||
if feed.youtube is True:
|
||||
new_posts = [entry for entry in newsFeed.entries if datetime.datetime.fromisoformat(entry.published) > yesterday_6am]
|
||||
else:
|
||||
new_posts = [entry for entry in newsFeed.entries if datetime.datetime.strptime(entry.published.replace('GMT', '+0000'), '%a, %d %b %Y %H:%M:%S %z') > yesterday_6am]
|
||||
|
||||
all_news_posts.extend(new_posts)
|
||||
|
||||
|
||||
random.shuffle(all_news_posts)
|
||||
|
||||
|
||||
for post in all_news_posts:
|
||||
linkURL = post["link"]
|
||||
title = post["title"]
|
||||
ftext = ""
|
||||
|
||||
LOG.info("Posting " + str(title))
|
||||
|
||||
if "summary" in post:
|
||||
ftext = html.unescape(post["summary"])
|
||||
# Using regular expressions to remove HTML tags
|
||||
ftext = re.sub('<[^<]+?>', '', ftext)
|
||||
pattern = r"L’article .* est apparu en premier sur .*"
|
||||
ftext = re.sub(pattern, '', ftext)
|
||||
|
||||
if "yt_videoid" in post:
|
||||
sub_stack_post.add({"type":"heading", "level":3, "content": title})
|
||||
videoId = post["yt_videoid"]
|
||||
sub_stack_post.add({"type":"youtube2", "src": videoId })
|
||||
sub_stack_post.add({'type': 'paragraph', 'content': [
|
||||
{'content': linkURL, 'marks': [{'type': "link", 'href': linkURL}]}]})
|
||||
else:
|
||||
|
||||
|
||||
|
||||
|
||||
if ftext != "":
|
||||
sub_stack_post.add({"type":"heading", "level":3, "content": title})
|
||||
sub_stack_post.add({"type":"paragraph", "content": ftext })
|
||||
sub_stack_post.add({'type': 'paragraph', 'content': [
|
||||
{'content': linkURL, 'marks': [{'type': "link", 'href': linkURL}]}]})
|
||||
|
||||
if "links" in post:
|
||||
for link in post["links"]:
|
||||
|
||||
if link["type"] == "image/jpg":
|
||||
imgUrl = link["href"]
|
||||
sub_stack_post.add({'type': 'captionedImage', 'src': imgUrl})
|
||||
|
||||
|
||||
sub_stack_post.add({"type":"horizontal_rule"})
|
||||
|
||||
|
||||
|
||||
sub_stack_post.add({"type":"heading", "level":3, "content": "Sources"})
|
||||
for feed in self.feeds:
|
||||
sub_stack_post.add({'type': 'paragraph', 'content': [
|
||||
{'content': feed.url, 'marks': [{'type': "link", 'href': feed.url}]}]})
|
||||
|
||||
|
||||
sub_stack_post.add({"type":"subscribeWidget", "message":"Abonnez-vous gratuitement pour recevoir chaque jour les news dans votre e-mail et soutenir mon travail."})
|
||||
|
||||
draft = self.api.post_draft(sub_stack_post.get_draft())
|
||||
self.api.prepublish_draft(draft.get("id"))
|
||||
self.api.publish_draft(draft.get("id"))
|
||||
|
||||
async def main(login, password, account):
|
||||
|
||||
setuplogger()
|
||||
|
||||
if os.path.exists("last_scan_date.txt"):
|
||||
with open("last_scan_date.txt", "r") as f:
|
||||
last_post_date = datetime.datetime.strptime(f.read().strip(), '%a, %d %b %Y %H:%M:%S %z')
|
||||
else:
|
||||
last_post_date = datetime.datetime.min.replace(tzinfo=datetime.timezone.utc)
|
||||
|
||||
feeds = []
|
||||
|
||||
feeds.append(RSSfeed("https://www.factornews.com/rss.xml"))
|
||||
feeds.append(RSSfeed("https://nofrag.com/feed"))
|
||||
feeds.append(RSSfeed("https://dystopeek.fr/feed/"))
|
||||
feeds.append(RSSfeed("https://thepixelpost.com/rss/"))
|
||||
feeds.append(RSSfeed("https://yamukass.substack.com/feed"))
|
||||
feeds.append(RSSfeed("https://tseret.com/categorie/tests/feed"))
|
||||
feeds.append(RSSfeed("https://www.gamesidestory.com/feed"))
|
||||
feeds.append(RSSfeed("https://www.nintendo-town.fr/feed"))
|
||||
feeds.append(RSSfeed("https://www.youtube.com/feeds/videos.xml?channel_id=UC-OvBDfZGn1OdsqMBwkOI_A", True))
|
||||
feeds.append(RSSfeed("https://www.youtube.com/feeds/videos.xml?playlist_id=PLZRiqJjIUlDTrwYs_UqEIts5fVaBpaIEz", True))
|
||||
|
||||
task = SubStackTask(login, password, account, feeds)
|
||||
|
||||
LOG.info("Starting bot")
|
||||
await task.run_daily_at_6_am()
|
||||
#await task.daily_task()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main("gael.honorez@gmail.com", "f3PaTGedjFc2gkr1ypi5", "https://aggregateurjvfr.substack.com"))
|
||||
-162
@@ -1,162 +0,0 @@
|
||||
# backfill_from_ghost.py
|
||||
from __future__ import annotations
|
||||
import os, re, sys, html
|
||||
from typing import Dict, List, Optional
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
# Reuse your existing GhostAdmin client (same headers/base/proxy behavior)
|
||||
# Adjust the import path if your Ghost client lives elsewhere.
|
||||
from presquegratos import GhostAdmin
|
||||
|
||||
from storage import Storage
|
||||
from keys import xgp_key, egs_key, psplus_key
|
||||
|
||||
# ---------------- Ghost helpers (reusing your admin client) ----------------
|
||||
def ghost_list_posts(ghost: GhostAdmin, page: int = 1) -> Dict:
|
||||
# Minimal params: avoid 'filter' and 'fields' to dodge 400 behind __bot proxy
|
||||
url = ghost.base + "posts/"
|
||||
params = {
|
||||
"limit": "50",
|
||||
"page": str(page),
|
||||
"order": "published_at DESC",
|
||||
"formats": "lexical,html", # <-- IMPORTANT
|
||||
}
|
||||
r = requests.get(url, headers=ghost._headers(), params=params, timeout=30)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
def list_recap_posts(ghost: GhostAdmin, hard_limit: int = 2000) -> List[Dict]:
|
||||
posts: List[Dict] = []
|
||||
page = 1
|
||||
while True:
|
||||
data = ghost_list_posts(ghost, page=page)
|
||||
batch = data.get("posts", [])
|
||||
if not batch:
|
||||
break
|
||||
# client-side filter to be robust to proxy quirks
|
||||
for p in batch:
|
||||
title = (p.get("title") or "").strip()
|
||||
if title.startswith("Récap hebdo"):
|
||||
posts.append(p)
|
||||
if len(batch) < 50 or len(posts) >= hard_limit:
|
||||
break
|
||||
page += 1
|
||||
return posts
|
||||
|
||||
# ---------------- Parsing helpers (unchanged) ----------------
|
||||
#MS_STORE_RE = re.compile(r"(?:microsoft|xbox)\.com/.*/store/.*/([0-9A-Z]{12,})", re.I)
|
||||
MS_STORE_RE = re.compile(r"(?:xbox|microsoft)\.com/.*/store/.*/([0-9A-Z]{12,16})", re.I)
|
||||
EPIC_RE = re.compile(r"epicgames\.com/store/.*/p/([\w\-]+)", re.I)
|
||||
PSBLOG_RE = re.compile(r"blog\.playstation\.com/.*", re.I)
|
||||
|
||||
def clean_text(s: str) -> str:
|
||||
return re.sub(r"\s+", " ", html.unescape(s or "")).strip()
|
||||
|
||||
def extract_sections(soup: BeautifulSoup) -> Dict[str, BeautifulSoup]:
|
||||
sections: Dict[str, BeautifulSoup] = {}
|
||||
current = None
|
||||
current_key = None
|
||||
for node in soup.find_all(["h2","h3","h4","p","ul","ol","div","section"]):
|
||||
if node.name in ("h2","h3","h4"):
|
||||
title = clean_text(node.get_text())
|
||||
key = None
|
||||
tl = title.lower()
|
||||
if "game pass" in tl:
|
||||
key = "xgp"
|
||||
elif "egs" in tl or "epic" in tl:
|
||||
key = "egs"
|
||||
elif "ps plus" in tl or "ps+" in tl:
|
||||
key = "psplus"
|
||||
if key:
|
||||
current_key = key
|
||||
current = sections[key] = soup.new_tag("div")
|
||||
continue
|
||||
if current_key and current is not None:
|
||||
current.append(node)
|
||||
return sections
|
||||
|
||||
def parse_xgp(section: BeautifulSoup) -> List[Dict]:
|
||||
items = []
|
||||
for a in section.find_all("a", href=True):
|
||||
href = a["href"]
|
||||
m = MS_STORE_RE.search(href)
|
||||
title = clean_text(a.get_text())
|
||||
if m or title:
|
||||
productId = m.group(1) if m else None
|
||||
items.append({"title": title, "productId": productId})
|
||||
uniq, seen = [], set()
|
||||
for it in items:
|
||||
k = xgp_key(it)
|
||||
if k not in seen:
|
||||
uniq.append(it); seen.add(k)
|
||||
return uniq
|
||||
|
||||
def parse_egs(section: BeautifulSoup) -> List[Dict]:
|
||||
items = []
|
||||
for a in section.find_all("a", href=True):
|
||||
if not EPIC_RE.search(a["href"]):
|
||||
continue
|
||||
title = clean_text(a.get_text()) or clean_text(a.get("title"))
|
||||
items.append({"title": title, "start": ""})
|
||||
uniq, seen = [], set()
|
||||
for it in items:
|
||||
k = egs_key(it)
|
||||
if k not in seen:
|
||||
uniq.append(it); seen.add(k)
|
||||
return uniq
|
||||
|
||||
def parse_psplus(section: BeautifulSoup, post_title: str) -> Optional[Dict]:
|
||||
a = section.find("a", href=PSBLOG_RE)
|
||||
url = a["href"] if a else ""
|
||||
m = re.search(r"(\d{2})-(\d{2})-(\d{4})", post_title)
|
||||
iso = ""
|
||||
if m:
|
||||
d, mth, y = m.group(1), m.group(2), m.group(3)
|
||||
iso = f"{y}-{mth}-{d}"
|
||||
return {"url": url, "date": iso}
|
||||
|
||||
# ---------------- Main backfill ----------------
|
||||
def backfill():
|
||||
# Use the same env your main script uses; GhostAdmin will read them internally or
|
||||
# you can pass them explicitly if your class expects (base_url, admin_key).
|
||||
ghost = GhostAdmin(
|
||||
admin_url=os.environ.get("GHOST_ADMIN_URL", "").rstrip("/") + "/",
|
||||
admin_key=os.environ.get("GHOST_ADMIN_KEY", "")
|
||||
)
|
||||
store = Storage()
|
||||
|
||||
posts = list_recap_posts(ghost)
|
||||
print(f"Found {len(posts)} recap posts.")
|
||||
|
||||
total_xgp = total_egs = total_ps = 0
|
||||
|
||||
dedup = []
|
||||
|
||||
for p in posts:
|
||||
|
||||
pid = p["id"]
|
||||
title = p.get("title") or ""
|
||||
html_body = p.get("html") or ""
|
||||
|
||||
soup = BeautifulSoup(html_body, "html.parser")
|
||||
sections = extract_sections(soup)
|
||||
|
||||
for it in parse_xgp(sections.get("xgp", BeautifulSoup("", "html.parser"))):
|
||||
key = xgp_key(it)
|
||||
if not key in dedup:
|
||||
store.remember("xgp", key, pid); total_xgp += 1
|
||||
dedup.append(key)
|
||||
|
||||
# for it in parse_egs(sections.get("egs", BeautifulSoup("", "html.parser"))):
|
||||
# store.remember("egs", egs_key(it), pid); total_egs += 1
|
||||
# if "psplus" in sections:
|
||||
# item = parse_psplus(sections["psplus"], title)
|
||||
# store.remember("psplus", psplus_key(item), pid); total_ps += 1
|
||||
|
||||
print(f"Backfilled from: {title}")
|
||||
|
||||
print(f"Done. Inserted ~ XGP:{total_xgp} | EGS:{total_egs} | PS+:{total_ps}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
backfill()
|
||||
@@ -1 +0,0 @@
|
||||
{"__cf_bm": "95up0icsYyESvD6suTUFG05xaWxwEr5_xuHUOv32G9I-1720025055-1.0.1.1-NlvsLW9j26FX8aPpLmVETEJ0zd.VyXefLr75kvT6iC.zHnPtkbIWgfesI0VaUGuvwV62qHpctJEoahLR9TIuHQ", "ab_experiment_sampled": "%22false%22", "ab_testing_id": "%22a6e7ba67-7dc0-452c-a935-d2f2bddd5edf%22", "ajs_anonymous_id": "%22e4535e95-1c5b-4173-82db-47807c57fb38%22", "cookie_storage_key": "f666a42c-49e8-47a2-bdbc-6eece0d6a06e", "substack.sid": "s%3ARLYSI2_XaTlGuYIpTYWjS8ib48PpuE0S.jNwCzcGzKUvUAuFdLNdfgxwewTUawIoDDZ05moubvzM", "visit_id": "%7B%22id%22%3A%22a0d46be8-56f4-406f-b1d7-14c41369b737%22%2C%22timestamp%22%3A%222024-07-03T16%3A44%3A13.349Z%22%7D", "AWSALBTG": "yw2xMbYVFbKWSzJiQsdCKp7mMH+wQ5T4/JIUc1TvywUi5iIJVXuO21AMhb+oPgegicdtpekLTDTl+zWKEekRsurS7+20skhmPxZXJf/Tl7jBd/PecbW7qa3DHkPvQtWz+SWD8+7P1rNjmY9lmyZgzH/ZeGgeiishRz9gsGO0OT/d", "AWSALBTGCORS": "yw2xMbYVFbKWSzJiQsdCKp7mMH+wQ5T4/JIUc1TvywUi5iIJVXuO21AMhb+oPgegicdtpekLTDTl+zWKEekRsurS7+20skhmPxZXJf/Tl7jBd/PecbW7qa3DHkPvQtWz+SWD8+7P1rNjmY9lmyZgzH/ZeGgeiishRz9gsGO0OT/d"}
|
||||
@@ -1,9 +0,0 @@
|
||||
version: '3.3'
|
||||
|
||||
services:
|
||||
substackjv:
|
||||
build: .
|
||||
volumes:
|
||||
- /path/to/your/host/directory:/data
|
||||
environment:
|
||||
- TZ=Europe/Brussels
|
||||
@@ -1,7 +0,0 @@
|
||||
set GHOST_ADMIN_KEY=68bad0e13546e700012dd65d:116a81b7e189d3b3d3b86082f97ef65daedb06498a3f1f902b8e0c08d095dc19
|
||||
set GHOST_ADMIN_URL=https://ghostadmin.zep.best/ghost/api/admin/__bot/FF4440EBA737506D397C170A8422109C357AA7582F10938B7C5F11D6B652F5D4
|
||||
set GHOST_EMAIL_SEGMENT=status:free
|
||||
set GHOST_NEWSLETTER_SLUG=default-newsletter
|
||||
set GHOST_CONTENT_URL=https://ghost.zep.best
|
||||
set DB_FILE_FALLBACK=f:\workspace\Substack_JV\data\published.db
|
||||
set MISTRAL_API_KEY=tQJHvYlmwz1ihKxOhXS3FmDNTRhBh6b3
|
||||
@@ -1,15 +0,0 @@
|
||||
https://www.factornews.com/rss.xml
|
||||
https://nofrag.com/feed
|
||||
https://dystopeek.fr/feed/
|
||||
https://thepixelpost.com/rss/
|
||||
https://yamukass.substack.com/feed
|
||||
https://tseret.com/categorie/tests/feed
|
||||
https://www.gamesidestory.com/feed
|
||||
https://www.nintendo-town.fr/feed
|
||||
https://jesuisungameur.com/feed
|
||||
https://www.switch-actu.fr/categorie/tests/tests-de-jeux/feed
|
||||
https://www.playscope.com/category/articles/test-gaming/feed
|
||||
https://jrpgfr.net/category/test/feed
|
||||
https://jv.jeuxonline.info/rss/dossiers/rss.xml
|
||||
https://www.youtube.com/feeds/videos.xml?channel_id=UC-OvBDfZGn1OdsqMBwkOI_A
|
||||
https://www.youtube.com/feeds/videos.xml?playlist_id=PLZRiqJjIUlDTrwYs_UqEIts5fVaBpaIEz
|
||||
@@ -1,20 +0,0 @@
|
||||
# keys.py (or inline in your main)
|
||||
def xgp_key(item) -> str:
|
||||
# Prefer stable Microsoft Store productId if present; fallback to normalized title.
|
||||
pid = (item.get("productId") or "").strip()
|
||||
if pid:
|
||||
return f"item:xgp:{pid}"
|
||||
title = (item.get("title") or "").strip().lower()
|
||||
return f"item:xgp:title:{title}"
|
||||
|
||||
def egs_key(item) -> str:
|
||||
# Use title + start window (your fetcher usually knows the free-week start)
|
||||
title = (item.get("title") or "").strip()
|
||||
start = (item.get("start") or "").strip() # ISO or YYYY-MM-DD
|
||||
return f"item:egs:{title}|{start}"
|
||||
|
||||
def psplus_key(item) -> str:
|
||||
# Use official PS Blog URL + the published month (or your computed date)
|
||||
url = (item.get("url") or "").strip()
|
||||
date = (item.get("date") or "").strip()
|
||||
return f"item:psplus:{url}|{date}"
|
||||
File diff suppressed because it is too large
Load Diff
-1040
File diff suppressed because it is too large
Load Diff
+2
-6
@@ -1,7 +1,3 @@
|
||||
requests
|
||||
feedparser
|
||||
PyJWT>=2.7,<3
|
||||
requests>=2.31
|
||||
feedparser>=6.0
|
||||
aiohttp
|
||||
bs4
|
||||
playwright
|
||||
python-substack
|
||||
-49
@@ -1,49 +0,0 @@
|
||||
# storage.py
|
||||
from __future__ import annotations
|
||||
import sqlite3, pathlib, datetime as dt
|
||||
from typing import Optional, Iterable, Tuple
|
||||
import os
|
||||
DB_PATH = "/data/published.db" # bind-mount ./data:/data in docker
|
||||
|
||||
_SCHEMA = """
|
||||
PRAGMA journal_mode = WAL;
|
||||
CREATE TABLE IF NOT EXISTS published_items(
|
||||
platform TEXT NOT NULL, -- e.g. xgp | egs | psplus
|
||||
key TEXT PRIMARY KEY, -- your dedupe key (see below)
|
||||
first_seen_utc TEXT NOT NULL, -- ISO-8601
|
||||
last_post_id TEXT -- Ghost post id that recorded it
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_platform ON published_items(platform);
|
||||
"""
|
||||
|
||||
class Storage:
|
||||
def __init__(self, db_path: str = DB_PATH):
|
||||
pathlib.Path(db_path).parent.mkdir(parents=True, exist_ok=True)
|
||||
self.conn = sqlite3.connect(db_path)
|
||||
self.conn.execute("PRAGMA foreign_keys = ON;")
|
||||
for stmt in filter(None, _SCHEMA.split(";")):
|
||||
if stmt.strip():
|
||||
self.conn.execute(stmt)
|
||||
|
||||
def seen(self, key: str) -> bool:
|
||||
cur = self.conn.execute("SELECT 1 FROM published_items WHERE key=?", (key,))
|
||||
return cur.fetchone() is not None
|
||||
|
||||
def remember(self, platform: str, key: str, post_id: Optional[str]):
|
||||
self.conn.execute(
|
||||
"INSERT OR IGNORE INTO published_items(platform,key,first_seen_utc,last_post_id) VALUES(?,?,?,?)",
|
||||
(platform, key, dt.datetime.utcnow().isoformat(), post_id),
|
||||
)
|
||||
if post_id:
|
||||
self.conn.execute("UPDATE published_items SET last_post_id=? WHERE key=?", (post_id, key))
|
||||
self.conn.commit()
|
||||
|
||||
def bulk_remember(self, platform: str, pairs: Iterable[Tuple[str, Optional[str]]]):
|
||||
rows = [(platform, k, dt.datetime.utcnow().isoformat(), pid) for (k, pid) in pairs]
|
||||
self.conn.executemany(
|
||||
"INSERT OR IGNORE INTO published_items(platform,key,first_seen_utc,last_post_id) VALUES(?,?,?,?)",
|
||||
rows
|
||||
)
|
||||
self.conn.commit()
|
||||
|
||||
|
||||
+5
-56
@@ -1,58 +1,7 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
#!/bin/bash
|
||||
|
||||
log() { printf '%s %s\n' "[$(date -u +%FT%TZ)]" "$*"; }
|
||||
# Pull the latest changes
|
||||
git pull origin main
|
||||
|
||||
stop() {
|
||||
log "stopping..."
|
||||
[ -n "${PID1-}" ] && kill -TERM "$PID1" 2>/dev/null || true
|
||||
[ -n "${PID2-}" ] && kill -TERM "$PID2" 2>/dev/null || true
|
||||
[ -n "${TPID-}" ] && kill -TERM "$TPID" 2>/dev/null || true
|
||||
wait || true
|
||||
exit 0
|
||||
}
|
||||
trap stop INT TERM
|
||||
|
||||
cd /app
|
||||
export GIT_TERMINAL_PROMPT=0
|
||||
|
||||
# MAJ forcée du code à chaque (re)démarrage
|
||||
if [ -d .git ]; then
|
||||
i=0
|
||||
while [ $i -lt 5 ]; do
|
||||
if git fetch --all --prune && git reset --hard origin/main; then
|
||||
log "git updated to origin/main"
|
||||
break
|
||||
fi
|
||||
i=$((i+1))
|
||||
log "git update failed (attempt $i/5); retrying in 10s..."
|
||||
sleep 10
|
||||
done
|
||||
[ $i -ge 5 ] && log "WARNING: git update failed after 5 attempts — continuing with current code"
|
||||
else
|
||||
log "WARNING: /app is not a git repo; skipping git update"
|
||||
fi
|
||||
|
||||
# Dossiers logs
|
||||
mkdir -p /var/log
|
||||
: > /var/log/daily.log
|
||||
: > /var/log/weekly.log
|
||||
|
||||
# Lancer les 2 bots (logs non bufferisés)
|
||||
python -u post_rss_to_ghost.py > /var/log/daily.log 2>&1 & PID1=$!
|
||||
python -u presquegratos.py > /var/log/weekly.log 2>&1 & PID2=$!
|
||||
|
||||
# Suivre les 2 fichiers de logs dans la sortie du conteneur
|
||||
tail -F /var/log/daily.log /var/log/weekly.log &
|
||||
TPID=$!
|
||||
|
||||
# Attente portable (pas de wait -n en /bin/sh)
|
||||
while :; do
|
||||
if ! kill -0 "$PID1" 2>/dev/null; then wait "$PID1" || true; break; fi
|
||||
if ! kill -0 "$PID2" 2>/dev/null; then wait "$PID2" || true; break; fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
# Si un des scripts sort, on arrête le tail (le trap TERM arrêtera l'autre script)
|
||||
kill -TERM "$TPID" 2>/dev/null || true
|
||||
wait || true
|
||||
# Run your Python script
|
||||
python Post_RSS_on_SubStack.py
|
||||
-23
@@ -1,23 +0,0 @@
|
||||
import feedparser
|
||||
import io
|
||||
import html
|
||||
import datetime
|
||||
import requests
|
||||
import time
|
||||
|
||||
url = r'https://www.xboxygen.com/spip.php?page=backend'
|
||||
|
||||
html_text = requests.get(url).text
|
||||
news = feedparser.parse(html_text)
|
||||
|
||||
yesterday_6am = datetime.datetime.now(datetime.timezone.utc).replace(hour=6, minute=0, second=0, microsecond=0) - datetime.timedelta(days=1)
|
||||
|
||||
try:
|
||||
new_posts = [entry for entry in news.entries if datetime.datetime.strptime(entry.published.replace('GMT', '+0000'), '%a, %d %b %Y %H:%M:%S %z') > yesterday_6am]
|
||||
|
||||
except:
|
||||
new_posts = [entry for entry in news.entries if datetime.datetime.fromtimestamp(time.mktime(entry.updated_parsed)).replace(tzinfo=datetime.timezone.utc) > yesterday_6am]
|
||||
#else if
|
||||
#entry.updated.replace('GMT', '+0000'), '%a, %d %b %Y %H:%M:%S %z'
|
||||
|
||||
print(new_posts)
|
||||
Reference in New Issue
Block a user