402 lines
16 KiB
Python
402 lines
16 KiB
Python
"""CDP WebSocket client for socialmag.
|
|
|
|
Talks to abang's already-running Brave over the Chrome DevTools Protocol
|
|
(default endpoint: http://127.0.0.1:9222). No browser is launched here — if
|
|
nothing is listening on 9222 we raise, loudly.
|
|
|
|
Shape:
|
|
|
|
cdp = CDPClient() # browser-level connection
|
|
await cdp.connect()
|
|
tab = await cdp.open_tab("https://www.threads.net/")
|
|
await tab.wait_for(SEL)
|
|
await tab.click(SEL)
|
|
await tab.close()
|
|
await cdp.close()
|
|
|
|
Every failure raises CDPError. Nothing here returns a plausible default.
|
|
"""
|
|
|
|
import asyncio
|
|
import base64
|
|
import json
|
|
import random
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
import websockets
|
|
|
|
DEFAULT_HTTP_ENDPOINT = "http://127.0.0.1:9336"
|
|
|
|
# Human-like pause between discrete actions (proposal: "random 0.5-2s").
|
|
HUMAN_DELAY_MIN = 0.5
|
|
HUMAN_DELAY_MAX = 1.5
|
|
|
|
# How long a wait_for polls, and how often.
|
|
DEFAULT_TIMEOUT = 20.0
|
|
POLL_INTERVAL = 0.25
|
|
|
|
|
|
class CDPError(RuntimeError):
|
|
"""Anything the browser side got wrong. Never swallowed, never defaulted."""
|
|
|
|
|
|
async def human_delay(lo=HUMAN_DELAY_MIN, hi=HUMAN_DELAY_MAX):
|
|
await asyncio.sleep(random.uniform(lo, hi))
|
|
|
|
|
|
def _http_get_json(url, timeout=5.0):
|
|
try:
|
|
with urllib.request.urlopen(url, timeout=timeout) as resp:
|
|
return json.loads(resp.read().decode("utf-8"))
|
|
except urllib.error.URLError as exc:
|
|
raise CDPError(
|
|
f"CDP endpoint unreachable at {url}: {exc}. "
|
|
"Is Brave running with --remote-debugging-port=9222?"
|
|
) from exc
|
|
except json.JSONDecodeError as exc:
|
|
raise CDPError(f"CDP endpoint {url} returned non-JSON: {exc}") from exc
|
|
|
|
|
|
class CDPClient:
|
|
"""Browser-level CDP connection. Owns the socket, hands out Tabs."""
|
|
|
|
def __init__(self, http_endpoint=DEFAULT_HTTP_ENDPOINT):
|
|
self.http_endpoint = http_endpoint.rstrip("/")
|
|
self._ws = None
|
|
self._reader_task = None
|
|
self._next_id = 0
|
|
self._pending = {} # message id -> Future
|
|
self._events = {} # session_id -> asyncio.Queue of (method, params)
|
|
self.browser_version = None
|
|
|
|
# ---- lifecycle -----------------------------------------------------
|
|
|
|
async def connect(self):
|
|
# NOTE: deliberately a plain blocking urlopen rather than
|
|
# asyncio.to_thread — `queue.py` in this package shadows the stdlib
|
|
# `queue` module, which concurrent.futures' thread pool imports. One
|
|
# 5s call at startup, before anything else is running.
|
|
version = _http_get_json(f"{self.http_endpoint}/json/version")
|
|
ws_url = version.get("webSocketDebuggerUrl")
|
|
if not ws_url:
|
|
raise CDPError(
|
|
f"/json/version has no webSocketDebuggerUrl: {version!r}"
|
|
)
|
|
self.browser_version = version.get("Browser")
|
|
self._ws = await websockets.connect(ws_url, max_size=None)
|
|
self._reader_task = asyncio.create_task(self._reader())
|
|
return self
|
|
|
|
async def close(self):
|
|
if self._reader_task:
|
|
self._reader_task.cancel()
|
|
try:
|
|
await self._reader_task
|
|
except asyncio.CancelledError:
|
|
pass
|
|
self._reader_task = None
|
|
if self._ws:
|
|
await self._ws.close()
|
|
self._ws = None
|
|
|
|
async def __aenter__(self):
|
|
return await self.connect()
|
|
|
|
async def __aexit__(self, *exc):
|
|
await self.close()
|
|
|
|
# ---- plumbing ------------------------------------------------------
|
|
|
|
async def _reader(self):
|
|
async for raw in self._ws:
|
|
msg = json.loads(raw)
|
|
mid = msg.get("id")
|
|
if mid is not None:
|
|
fut = self._pending.pop(mid, None)
|
|
if fut and not fut.done():
|
|
fut.set_result(msg)
|
|
continue
|
|
session_id = msg.get("sessionId")
|
|
queue = self._events.get(session_id)
|
|
if queue is not None:
|
|
queue.put_nowait((msg.get("method"), msg.get("params", {})))
|
|
|
|
async def send(self, method, params=None, session_id=None, timeout=30.0):
|
|
if self._ws is None:
|
|
raise CDPError("CDPClient.send() before connect()")
|
|
self._next_id += 1
|
|
mid = self._next_id
|
|
payload = {"id": mid, "method": method, "params": params or {}}
|
|
if session_id:
|
|
payload["sessionId"] = session_id
|
|
fut = asyncio.get_running_loop().create_future()
|
|
self._pending[mid] = fut
|
|
await self._ws.send(json.dumps(payload))
|
|
try:
|
|
msg = await asyncio.wait_for(fut, timeout)
|
|
except asyncio.TimeoutError as exc:
|
|
self._pending.pop(mid, None)
|
|
raise CDPError(f"CDP call timed out after {timeout}s: {method}") from exc
|
|
if "error" in msg:
|
|
raise CDPError(f"CDP {method} failed: {msg['error']}")
|
|
return msg.get("result", {})
|
|
|
|
# ---- tabs ----------------------------------------------------------
|
|
|
|
async def list_targets(self):
|
|
result = await self.send("Target.getTargets")
|
|
return [t for t in result["targetInfos"] if t["type"] == "page"]
|
|
|
|
async def _attach(self, target_id):
|
|
result = await self.send(
|
|
"Target.attachToTarget", {"targetId": target_id, "flatten": True}
|
|
)
|
|
session_id = result["sessionId"]
|
|
self._events[session_id] = asyncio.Queue()
|
|
tab = Tab(self, target_id, session_id)
|
|
await tab.send("Page.enable")
|
|
await tab.send("Runtime.enable")
|
|
await tab.send("DOM.enable")
|
|
return tab
|
|
|
|
async def open_tab(self, url):
|
|
"""Open a fresh tab and attach to it."""
|
|
result = await self.send("Target.createTarget", {"url": url})
|
|
tab = await self._attach(result["targetId"])
|
|
await tab.wait_for_load()
|
|
return tab
|
|
|
|
async def attach_existing(self, url_contains):
|
|
"""Attach to an already-open tab whose URL contains `url_contains`."""
|
|
for info in await self.list_targets():
|
|
if url_contains in info.get("url", ""):
|
|
return await self._attach(info["targetId"])
|
|
raise CDPError(f"no open tab whose URL contains {url_contains!r}")
|
|
|
|
async def get_or_open_tab(self, url, url_contains):
|
|
"""Reuse a warm existing tab if one matches, otherwise open a fresh one.
|
|
Returns (tab, reused: bool). Caller should skip tab.close() when reused."""
|
|
try:
|
|
tab = await self.attach_existing(url_contains)
|
|
return tab, True
|
|
except CDPError:
|
|
tab = await self.open_tab(url)
|
|
return tab, False
|
|
|
|
def _drop_session(self, session_id):
|
|
self._events.pop(session_id, None)
|
|
|
|
|
|
class Tab:
|
|
"""One page target. All the verbs the platform modules use."""
|
|
|
|
def __init__(self, client, target_id, session_id):
|
|
self.client = client
|
|
self.target_id = target_id
|
|
self.session_id = session_id
|
|
|
|
async def send(self, method, params=None, timeout=30.0):
|
|
return await self.client.send(
|
|
method, params, session_id=self.session_id, timeout=timeout
|
|
)
|
|
|
|
async def close(self):
|
|
await self.client.send("Target.closeTarget", {"targetId": self.target_id})
|
|
self.client._drop_session(self.session_id)
|
|
|
|
# ---- navigation ----------------------------------------------------
|
|
|
|
async def navigate(self, url, timeout=DEFAULT_TIMEOUT):
|
|
result = await self.send("Page.navigate", {"url": url})
|
|
if result.get("errorText"):
|
|
raise CDPError(f"navigate to {url} failed: {result['errorText']}")
|
|
await self.wait_for_load(timeout)
|
|
|
|
async def wait_for_load(self, timeout=DEFAULT_TIMEOUT):
|
|
"""Wait for document.readyState == 'complete' (poll; simpler than
|
|
racing Page.loadEventFired against an already-loaded target)."""
|
|
deadline = asyncio.get_running_loop().time() + timeout
|
|
while asyncio.get_running_loop().time() < deadline:
|
|
state = await self.eval("document.readyState")
|
|
if state == "complete":
|
|
return
|
|
await asyncio.sleep(POLL_INTERVAL)
|
|
raise CDPError(f"page did not finish loading within {timeout}s: {await self.url()}")
|
|
|
|
async def url(self):
|
|
return await self.eval("location.href")
|
|
|
|
# ---- evaluation ----------------------------------------------------
|
|
|
|
async def eval(self, expression, await_promise=False):
|
|
"""Evaluate JS, return the value. Throws loudly on JS exceptions."""
|
|
result = await self.send(
|
|
"Runtime.evaluate",
|
|
{
|
|
"expression": expression,
|
|
"returnByValue": True,
|
|
"awaitPromise": await_promise,
|
|
"userGesture": True,
|
|
},
|
|
)
|
|
if "exceptionDetails" in result:
|
|
detail = result["exceptionDetails"]
|
|
text = detail.get("exception", {}).get("description") or detail.get("text")
|
|
raise CDPError(f"JS threw: {text}\n expression: {expression[:200]}")
|
|
return result["result"].get("value")
|
|
|
|
# ---- querying / waiting -------------------------------------------
|
|
|
|
async def exists(self, selector):
|
|
return bool(await self.eval(f"!!document.querySelector({json.dumps(selector)})"))
|
|
|
|
async def wait_for(self, selector, timeout=DEFAULT_TIMEOUT, visible=True):
|
|
"""Poll until selector is present (and, by default, has a box)."""
|
|
check = (
|
|
"(() => { const e = document.querySelector(%s); if (!e) return false;"
|
|
" if (!%s) return true; const r = e.getBoundingClientRect();"
|
|
" return r.width > 0 && r.height > 0; })()"
|
|
% (json.dumps(selector), "true" if visible else "false")
|
|
)
|
|
deadline = asyncio.get_running_loop().time() + timeout
|
|
while asyncio.get_running_loop().time() < deadline:
|
|
if await self.eval(check):
|
|
return True
|
|
await asyncio.sleep(POLL_INTERVAL)
|
|
raise CDPError(f"timed out after {timeout}s waiting for selector: {selector}")
|
|
|
|
async def wait_for_gone(self, selector, timeout=DEFAULT_TIMEOUT):
|
|
"""Poll until selector disappears — how every dialog flow confirms success."""
|
|
deadline = asyncio.get_running_loop().time() + timeout
|
|
while asyncio.get_running_loop().time() < deadline:
|
|
if not await self.exists(selector):
|
|
return True
|
|
await asyncio.sleep(POLL_INTERVAL)
|
|
raise CDPError(f"timed out after {timeout}s waiting for {selector} to disappear")
|
|
|
|
async def wait_for_text(self, text, timeout=DEFAULT_TIMEOUT):
|
|
"""Poll until the text appears anywhere in document.body."""
|
|
expr = f"document.body.innerText.includes({json.dumps(text)})"
|
|
deadline = asyncio.get_running_loop().time() + timeout
|
|
while asyncio.get_running_loop().time() < deadline:
|
|
if await self.eval(expr):
|
|
return True
|
|
await asyncio.sleep(POLL_INTERVAL)
|
|
raise CDPError(f"timed out after {timeout}s waiting for text: {text!r}")
|
|
|
|
# ---- interaction ---------------------------------------------------
|
|
|
|
async def _box(self, selector):
|
|
box = await self.eval(
|
|
"(() => { const e = document.querySelector(%s); if (!e) return null;"
|
|
" e.scrollIntoView({block:'center'});"
|
|
" const r = e.getBoundingClientRect();"
|
|
" return {x: r.left + r.width/2, y: r.top + r.height/2,"
|
|
" w: r.width, h: r.height}; })()" % json.dumps(selector)
|
|
)
|
|
if not box:
|
|
raise CDPError(f"element not found: {selector}")
|
|
if box["w"] <= 0 or box["h"] <= 0:
|
|
raise CDPError(f"element has zero size (not visible): {selector}")
|
|
return box
|
|
|
|
async def click(self, selector, delay=True):
|
|
"""Real mouse click at the element's centre — the human-shaped path."""
|
|
box = await self._box(selector)
|
|
await asyncio.sleep(0.1)
|
|
for kind in ("mousePressed", "mouseReleased"):
|
|
await self.send(
|
|
"Input.dispatchMouseEvent",
|
|
{
|
|
"type": kind,
|
|
"x": box["x"],
|
|
"y": box["y"],
|
|
"button": "left",
|
|
"clickCount": 1,
|
|
},
|
|
)
|
|
await asyncio.sleep(random.uniform(0.04, 0.12))
|
|
if delay:
|
|
await human_delay()
|
|
|
|
async def click_js(self, expression, delay=True):
|
|
"""Click via JS on an element the expression returns.
|
|
|
|
Facebook needs this: the composer's real click target is an ancestor
|
|
[role=button], and mouse-clicking the visible span opens Messenger
|
|
instead of the composer.
|
|
"""
|
|
ok = await self.eval(
|
|
"(() => { const el = (%s); if (!el) return false; el.click(); return true; })()"
|
|
% expression
|
|
)
|
|
if not ok:
|
|
raise CDPError(f"click_js found no element for expression: {expression[:200]}")
|
|
if delay:
|
|
await human_delay()
|
|
|
|
async def focus(self, selector):
|
|
await self.eval(
|
|
"(() => { const e = document.querySelector(%s);"
|
|
" if (!e) throw new Error('focus: not found'); e.focus(); })()"
|
|
% json.dumps(selector)
|
|
)
|
|
|
|
async def type_text(self, selector, text, per_char=False, delay=True):
|
|
"""Focus `selector` then type. Default is Input.insertText (fast, and
|
|
React contenteditables accept it); per_char sends real key events for
|
|
editors that only listen to keydown."""
|
|
await self.click(selector, delay=False)
|
|
await asyncio.sleep(0.2)
|
|
if per_char:
|
|
for ch in text:
|
|
await self.send("Input.dispatchKeyEvent", {"type": "keyDown", "text": ch})
|
|
await self.send("Input.dispatchKeyEvent", {"type": "keyUp", "text": ch})
|
|
await asyncio.sleep(random.uniform(0.02, 0.09))
|
|
else:
|
|
await self.send("Input.insertText", {"text": text})
|
|
if delay:
|
|
await human_delay()
|
|
|
|
async def press_key(self, key, code=None, key_code=None):
|
|
params = {"type": "keyDown", "key": key}
|
|
if code:
|
|
params["code"] = code
|
|
if key_code:
|
|
params["windowsVirtualKeyCode"] = key_code
|
|
params["nativeVirtualKeyCode"] = key_code
|
|
await self.send("Input.dispatchKeyEvent", params)
|
|
await self.send("Input.dispatchKeyEvent", dict(params, type="keyUp"))
|
|
|
|
# ---- file inputs ---------------------------------------------------
|
|
|
|
async def set_file_input(self, selector, paths):
|
|
"""DOM.setFileInputFiles — injects a file without the native picker.
|
|
|
|
`selector` must resolve to an <input type=file>. Paths must be
|
|
absolute and must exist (caller validates; we re-check the shape).
|
|
"""
|
|
if isinstance(paths, str):
|
|
paths = [paths]
|
|
doc = await self.send("DOM.getDocument", {"depth": 0})
|
|
node = await self.send(
|
|
"DOM.querySelector",
|
|
{"nodeId": doc["root"]["nodeId"], "selector": selector},
|
|
)
|
|
node_id = node.get("nodeId")
|
|
if not node_id:
|
|
raise CDPError(f"no file input matched selector: {selector}")
|
|
await self.send("DOM.setFileInputFiles", {"files": paths, "nodeId": node_id})
|
|
await human_delay()
|
|
|
|
# ---- diagnostics ---------------------------------------------------
|
|
|
|
async def screenshot(self, path):
|
|
"""Capture the page. Used on failure so a broken flow leaves a trace."""
|
|
result = await self.send("Page.captureScreenshot", {"format": "png"})
|
|
data = base64.b64decode(result["data"])
|
|
with open(path, "wb") as fh:
|
|
fh.write(data)
|
|
return path
|