v0.5: capture lane, sqlite store, refresher wired to both
capture.py — read the platform's own JSON instead of the pixels it painted. Network capture on X (/i/api/graphql HomeLatestTimeline) and Threads (/graphql/query feedData); in-document read on Instagram, whose feed is prefetched into a RelayPrefetchedStreamCache script and never crosses the wire. 190 X rows in 0.9s against the DOM lane's 10 in 11.1s, with exact integer counts and view numbers the DOM never exposes. Waits for Network.loadingFinished, not responseReceived — the body is not there yet at responseReceived. Nothing captured raises; an empty list would say the platform posted nothing when the truth is our listener missed it. store.py — SQLite, stdlib only. platform_checks / items / metrics. id = sha1(platform|kind|url or text), first_seen written once and ts under COALESCE so a derived Threads marker cannot decay across passes. FEED_PER_PLATFORM = 10 caps any one platform before the merge, or X's ~185 rows a pass sort Instagram's 3 off the page entirely. dashboard.json is now a derived view of the database; dashboard.html did not have to change for it. refresher.py — check_all runs the background-safe platforms in parallel (19.0s sequential to 6.1s), tiktok still sequential and foreground. --skip-foreground leaves it out of a timed pass and store.latest_checks keeps returning its last real result with its own older checkedAt; stale is honest, a fabricated row is not. Browser down stays the third state and exits 0. .gitignore: socialmag.db (regenerable), deploy/kdmirah/ (carries the box's VNC password and addresses — stays in the lab, out of git). Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
@@ -0,0 +1,322 @@
|
||||
"""socialmag.store — the durable half. SQLite, stdlib only, one file.
|
||||
|
||||
Not here for size. `dashboard.json` is 4KB and would hold that forever. It is
|
||||
here because **every refresh replaced `feed` and `notifications` wholesale**, so:
|
||||
|
||||
* there was no history — "how often does a session drop?", the exact question
|
||||
README's VPS section says we need answered, was unanswerable;
|
||||
* there was no concept of *new* — three Threads rows aged `11h` to `1d` and
|
||||
nothing knew they were the same three rows;
|
||||
* 🔴 the derived `ts` decayed. A row sank an hour every hour.
|
||||
|
||||
`first_seen` fixes the third one outright: it is a timestamp **we own**, exact
|
||||
and fixed. Threads' missing clock stops being a problem instead of being patched.
|
||||
|
||||
from socialmag import store
|
||||
db = store.open_db()
|
||||
store.record_checks(db, rows)
|
||||
store.record_items(db, "feed", captured_rows)
|
||||
data = store.snapshot(db, previous) # -> the dashboard.json view
|
||||
|
||||
`dashboard.json` becomes a **derived view**, so `dashboard.html` does not change.
|
||||
The database is the record; the JSON is a picture of it.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import time
|
||||
|
||||
DB_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "socialmag.db")
|
||||
|
||||
#: How many rows of each kind the derived dashboard.json carries. The database
|
||||
#: keeps everything; this is only what the page is asked to render.
|
||||
VIEW_LIMIT = 25
|
||||
|
||||
#: Most rows one platform may contribute to the feed view. X returns ~185 fresh
|
||||
#: rows a pass, Instagram 3 older ones — without a cap the "cross-platform feed"
|
||||
#: is an X feed with the others sorted off the end.
|
||||
FEED_PER_PLATFORM = 10
|
||||
|
||||
SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS platform_checks (
|
||||
platform TEXT NOT NULL,
|
||||
checked_at TEXT NOT NULL,
|
||||
logged_in INTEGER, -- 1 / 0, NULL when the check itself failed
|
||||
error TEXT, -- set only for the third state
|
||||
PRIMARY KEY (platform, checked_at)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS items (
|
||||
id TEXT PRIMARY KEY, -- sha1(platform|kind|url or text)
|
||||
platform TEXT NOT NULL,
|
||||
kind TEXT NOT NULL, -- 'feed' | 'notification' | 'post'
|
||||
author TEXT,
|
||||
text TEXT,
|
||||
ts TEXT, -- the platform's own time, when it gives one
|
||||
url TEXT,
|
||||
likes INTEGER,
|
||||
replies INTEGER,
|
||||
reposts INTEGER,
|
||||
views INTEGER,
|
||||
first_seen TEXT NOT NULL, -- ours. exact, fixed, never guessed
|
||||
last_seen TEXT NOT NULL,
|
||||
times_seen INTEGER NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS items_by_kind ON items(kind, first_seen DESC);
|
||||
|
||||
-- One row per item per pass where a count actually moved. This is the
|
||||
-- "snapshot history" a paid product sells; it costs a WHERE clause.
|
||||
CREATE TABLE IF NOT EXISTS metrics (
|
||||
item_id TEXT NOT NULL,
|
||||
seen_at TEXT NOT NULL,
|
||||
likes INTEGER,
|
||||
replies INTEGER,
|
||||
reposts INTEGER,
|
||||
views INTEGER,
|
||||
PRIMARY KEY (item_id, seen_at)
|
||||
);
|
||||
"""
|
||||
|
||||
COUNTS = ("likes", "replies", "reposts", "views")
|
||||
|
||||
|
||||
def now():
|
||||
return time.strftime("%Y-%m-%dT%H:%M:%S%z")
|
||||
|
||||
|
||||
def open_db(path=DB_PATH):
|
||||
db = sqlite3.connect(path)
|
||||
db.row_factory = sqlite3.Row
|
||||
db.executescript(SCHEMA)
|
||||
return db
|
||||
|
||||
|
||||
def item_id(platform, kind, row):
|
||||
"""Stable identity for a row that the platform may not have given one.
|
||||
|
||||
Prefer the platform's own id, then the permalink, then the text. Text is the
|
||||
last resort and it is the reason Threads' three unchanging notification rows
|
||||
collapse to three records instead of three per pass.
|
||||
"""
|
||||
seed = row.get("id") or row.get("url") or row.get("text") or ""
|
||||
if not seed:
|
||||
raise ValueError(f"store: {platform} {kind} row has no id, url or text: {row!r}")
|
||||
digest = hashlib.sha1(f"{platform}|{kind}|{seed}".encode("utf-8"))
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# writes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def record_checks(db, rows):
|
||||
"""One login-state row per platform per pass. Three states, all kept.
|
||||
|
||||
`rows` is refresher's {platform: {loggedIn|error, checkedAt}}.
|
||||
"""
|
||||
stamp = now()
|
||||
with db:
|
||||
for platform, row in rows.items():
|
||||
logged_in = row.get("loggedIn")
|
||||
db.execute(
|
||||
"INSERT OR REPLACE INTO platform_checks"
|
||||
" (platform, checked_at, logged_in, error) VALUES (?, ?, ?, ?)",
|
||||
(platform,
|
||||
row.get("checkedAt") or stamp,
|
||||
None if logged_in is None else int(bool(logged_in)),
|
||||
row.get("error")),
|
||||
)
|
||||
return len(rows)
|
||||
|
||||
|
||||
def record_items(db, kind, rows):
|
||||
"""Upsert rows, and append a metrics snapshot only when a count moved.
|
||||
|
||||
Returns (new, updated). A row already in the table keeps its `first_seen` —
|
||||
that is the whole point of the table.
|
||||
"""
|
||||
stamp = now()
|
||||
new = updated = 0
|
||||
with db:
|
||||
for row in rows:
|
||||
platform = row.get("platform")
|
||||
if not platform:
|
||||
raise ValueError(f"store: {kind} row has no platform: {row!r}")
|
||||
key = item_id(platform, kind, row)
|
||||
counts = {c: row.get(c) for c in COUNTS}
|
||||
existing = db.execute(
|
||||
"SELECT likes, replies, reposts, views, times_seen FROM items WHERE id = ?",
|
||||
(key,),
|
||||
).fetchone()
|
||||
|
||||
if existing is None:
|
||||
db.execute(
|
||||
"INSERT INTO items (id, platform, kind, author, text, ts, url,"
|
||||
" likes, replies, reposts, views, first_seen, last_seen, times_seen)"
|
||||
" VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,1)",
|
||||
(key, platform, kind, row.get("author"), row.get("text"),
|
||||
row.get("ts"), row.get("url"),
|
||||
counts["likes"], counts["replies"], counts["reposts"], counts["views"],
|
||||
stamp, stamp),
|
||||
)
|
||||
new += 1
|
||||
moved = any(v is not None for v in counts.values())
|
||||
else:
|
||||
# 🔴 `ts` is written once and never rewritten. A Threads
|
||||
# notification's time is derived from a relative marker ("1d"),
|
||||
# so re-deriving it every pass moves the row forward an hour
|
||||
# every hour — exactly the decay this table exists to stop. The
|
||||
# first derivation was made closest to first sighting and is the
|
||||
# most accurate one we will ever have.
|
||||
db.execute(
|
||||
"UPDATE items SET author=?, text=?, ts=COALESCE(ts, ?), url=?,"
|
||||
" likes=?, replies=?, reposts=?, views=?,"
|
||||
" last_seen=?, times_seen=times_seen+1 WHERE id=?",
|
||||
(row.get("author"), row.get("text"), row.get("ts"), row.get("url"),
|
||||
counts["likes"], counts["replies"], counts["reposts"], counts["views"],
|
||||
stamp, key),
|
||||
)
|
||||
updated += 1
|
||||
moved = any(counts[c] is not None and counts[c] != existing[c]
|
||||
for c in COUNTS)
|
||||
|
||||
if moved:
|
||||
db.execute(
|
||||
"INSERT OR REPLACE INTO metrics"
|
||||
" (item_id, seen_at, likes, replies, reposts, views)"
|
||||
" VALUES (?,?,?,?,?,?)",
|
||||
(key, stamp, counts["likes"], counts["replies"],
|
||||
counts["reposts"], counts["views"]),
|
||||
)
|
||||
return new, updated
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# reads
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def latest_checks(db):
|
||||
"""The newest check per platform, in refresher's own row shape."""
|
||||
rows = {}
|
||||
for r in db.execute(
|
||||
"SELECT platform, checked_at, logged_in, error FROM platform_checks"
|
||||
" WHERE (platform, checked_at) IN"
|
||||
" (SELECT platform, MAX(checked_at) FROM platform_checks GROUP BY platform)"
|
||||
):
|
||||
row = {"checkedAt": r["checked_at"]}
|
||||
if r["error"]:
|
||||
row["error"] = r["error"]
|
||||
else:
|
||||
row["loggedIn"] = bool(r["logged_in"])
|
||||
rows[r["platform"]] = row
|
||||
return rows
|
||||
|
||||
|
||||
def recent_items(db, kind, limit=VIEW_LIMIT, per_platform=None):
|
||||
"""Newest first by the platform's own time, falling back to when we met it.
|
||||
|
||||
A row with no `ts` — every Threads notification — sorts by `first_seen`,
|
||||
which is exact rather than derived from a decaying "1d" marker.
|
||||
|
||||
`per_platform` caps how many rows any one platform may contribute before
|
||||
the merge. Without it a pure time sort is not a cross-platform feed: X
|
||||
returns ~185 fresh rows a pass against Instagram's 3 older ones, so IG never
|
||||
reaches the page at all. The cap is representation, not ranking — rows still
|
||||
sort by time against each other afterwards.
|
||||
"""
|
||||
if per_platform:
|
||||
platforms = [r["platform"] for r in db.execute(
|
||||
"SELECT DISTINCT platform FROM items WHERE kind = ?", (kind,))]
|
||||
rows = []
|
||||
for platform in platforms:
|
||||
rows.extend(db.execute(
|
||||
"SELECT * FROM items WHERE kind = ? AND platform = ?"
|
||||
" ORDER BY COALESCE(ts, first_seen) DESC LIMIT ?",
|
||||
(kind, platform, per_platform),
|
||||
).fetchall())
|
||||
rows.sort(key=lambda r: r["ts"] or r["first_seen"], reverse=True)
|
||||
rows = rows[:limit]
|
||||
else:
|
||||
rows = db.execute(
|
||||
"SELECT * FROM items WHERE kind = ?"
|
||||
" ORDER BY COALESCE(ts, first_seen) DESC LIMIT ?",
|
||||
(kind, limit),
|
||||
).fetchall()
|
||||
|
||||
out = []
|
||||
for r in rows:
|
||||
row = {
|
||||
"platform": r["platform"], "author": r["author"], "text": r["text"],
|
||||
"url": r["url"], "firstSeen": r["first_seen"], "timesSeen": r["times_seen"],
|
||||
}
|
||||
if r["ts"]:
|
||||
row["ts"] = r["ts"]
|
||||
else:
|
||||
# No platform clock. Say which stamp this is instead of passing our
|
||||
# own off as theirs.
|
||||
row["ts"] = r["first_seen"]
|
||||
row["_tsIsFirstSeen"] = True
|
||||
for c in COUNTS:
|
||||
if r[c] is not None:
|
||||
row[c] = r[c]
|
||||
out.append(row)
|
||||
return out
|
||||
|
||||
|
||||
def uptime(db, platform, days=7):
|
||||
"""(checks, drops) for one platform — the number the VPS question needs.
|
||||
|
||||
A "drop" is a check that came back signed-out. Errors are not drops: a
|
||||
browser that was not running says nothing about the account.
|
||||
"""
|
||||
since = time.strftime(
|
||||
"%Y-%m-%dT%H:%M:%S%z", time.localtime(time.time() - days * 86400)
|
||||
)
|
||||
row = db.execute(
|
||||
"SELECT COUNT(*) AS checks,"
|
||||
" SUM(CASE WHEN logged_in = 0 THEN 1 ELSE 0 END) AS drops"
|
||||
" FROM platform_checks WHERE platform = ? AND checked_at >= ?"
|
||||
" AND error IS NULL",
|
||||
(platform, since),
|
||||
).fetchone()
|
||||
return row["checks"], (row["drops"] or 0)
|
||||
|
||||
|
||||
def snapshot(db, previous=None, limit=VIEW_LIMIT):
|
||||
"""Build the dashboard.json view from the database.
|
||||
|
||||
`previous` carries forward keys the store does not own — `posts`, which only
|
||||
a real create_post can produce, and `_note`.
|
||||
"""
|
||||
data = dict(previous or {})
|
||||
data["platforms"] = latest_checks(db)
|
||||
data["feed"] = recent_items(db, "feed", limit, per_platform=FEED_PER_PLATFORM)
|
||||
data["notifications"] = recent_items(db, "notification", limit)
|
||||
data.setdefault("posts", [])
|
||||
return data
|
||||
|
||||
|
||||
def stats(db):
|
||||
"""One line per table — what is actually in there."""
|
||||
return {
|
||||
"checks": db.execute("SELECT COUNT(*) FROM platform_checks").fetchone()[0],
|
||||
"items": db.execute("SELECT COUNT(*) FROM items").fetchone()[0],
|
||||
"metrics": db.execute("SELECT COUNT(*) FROM metrics").fetchone()[0],
|
||||
"kinds": {
|
||||
r["kind"]: r["n"] for r in db.execute(
|
||||
"SELECT kind, COUNT(*) AS n FROM items GROUP BY kind"
|
||||
)
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
db = open_db()
|
||||
print(json.dumps(stats(db), indent=2))
|
||||
for platform in sorted({r["platform"] for r in db.execute(
|
||||
"SELECT DISTINCT platform FROM platform_checks")}):
|
||||
checks, drops = uptime(db, platform)
|
||||
print(f" {platform:<8} {checks:>4} checks, {drops} signed-out")
|
||||
Reference in New Issue
Block a user