Pull feed

OP=$(uuidgen)
curl -m 30 --retry 3 --retry-all-errors https://thread.page/api/integrations/v1/subscriptions \
  -H "Authorization: Bearer $THREADPAGE_TOKEN" \
  -H "Operation-Id: $OP" \
  -H "Content-Type: application/json" \
  -d '{"kind": "pull", "event_types": ["note.created", "note.updated"]}'

curl -m 40 "https://thread.page/api/integrations/v1/subscriptions/$FEED_ID/events?after=$CURSOR&wait=25" \
  -H "Authorization: Bearer $THREADPAGE_TOKEN"

curl -m 30 "https://thread.page/api/integrations/v1/subscriptions/$SUBSCRIPTION_ID/events" \
  -H "Authorization: Bearer $THREADPAGE_TOKEN"

// A pull feed reader: it reads events as they come, handles each one once, and keeps its place
// in a file, so it continues where it stopped after a restart.
// Run with Node.js 24: THREADPAGE_TOKEN=tp_… node feed_reader.mjs
// Set THREADPAGE_FEED_ID to read a feed you already have; otherwise it creates one.
import { randomUUID } from 'node:crypto';
import { readFile, rename, writeFile } from 'node:fs/promises';
import { setTimeout as sleep } from 'node:timers/promises';

const API = 'https://thread.page/api/integrations/v1';
const AUTH = { Authorization: `Bearer ${process.env.THREADPAGE_TOKEN}` };
const STATE = process.env.THREADPAGE_FEED_STATE ?? 'threadpage-feed.json';

async function load() {
  try {
    return JSON.parse(await readFile(STATE, 'utf8'));
  } catch (error) {
    if (error.code === 'ENOENT') return { seen: [] };
    throw error;
  }
}

async function save(state) {
  await writeFile(`${STATE}.tmp`, JSON.stringify(state));
  await rename(`${STATE}.tmp`, STATE); // a crash never leaves half a file
}

// POST /subscriptions with one Operation-Id, kept in the state file until the feed exists.
async function createFeed(state) {
  state.createOperationId ??= randomUUID();
  await save(state);
  let answer;
  for (let attempt = 1; ; attempt++) {
    try {
      answer = await fetch(`${API}/subscriptions`, {
        method: 'POST',
        headers: { ...AUTH, 'Operation-Id': state.createOperationId, 'Content-Type': 'application/json' },
        body: JSON.stringify({ kind: 'pull', event_types: ['note.created', 'note.updated', 'note.trashed'] }),
        signal: AbortSignal.timeout(30_000),
      });
      break;
    } catch (error) {
      if (attempt === 3) throw error;
      await sleep(attempt * 2_000);
    }
  }
  if (!answer.ok) throw new Error(`${answer.status}: ${await answer.text()}`);
  state.feedId = (await answer.json()).subscription.id;
  delete state.createOperationId;
  await save(state);
}

function handle(event) {
  const item = event.data.note ?? event.data.thread ?? {};
  console.log(event.type, item.id);
}

async function readForever(state) {
  for (;;) {
    const url = new URL(`${API}/subscriptions/${state.feedId}/events`);
    if (state.cursor) {
      url.searchParams.set('after', state.cursor);
      url.searchParams.set('wait', '25');
    }
    let answer;
    try {
      answer = await fetch(url, { headers: AUTH, signal: AbortSignal.timeout(40_000) });
    } catch {
      await sleep(5_000);
      continue;
    }
    if (answer.status === 410) { // the cursor is older than 7 days
      state.cursor = (await answer.json()).details.resync_cursor;
      await save(state);
      console.log('Events in between are not replayed: read current notes through the API.');
      continue;
    }
    if (answer.status === 429 || answer.status === 503) {
      await sleep(Number(answer.headers.get('Retry-After') ?? 5) * 1_000);
      continue;
    }
    if (answer.status === 409) { // the feed is paused; its events wait
      await sleep(60_000);
      continue;
    }
    if (!answer.ok) throw new Error(`${answer.status}: ${await answer.text()}`);
    const page = await answer.json();
    for (const event of page.events) {
      if (state.seen.includes(event.id)) continue; // the same event again
      handle(event);
      state.seen = [...state.seen, event.id].slice(-1000);
      await save(state);
    }
    state.cursor = page.cursor; // only once the events are handled
    await save(state);
  }
}

let state = await load();
const feedId = process.env.THREADPAGE_FEED_ID;
if (feedId && state.feedId !== feedId) state = { seen: [], feedId }; // a cursor belongs to one feed
if (!state.feedId) await createFeed(state);
await readForever(state);

# A pull feed reader: it reads events as they come, handles each one once, and keeps its place
# in a file, so it continues where it stopped after a restart.
# Run with Python 3.13: pip install requests, then THREADPAGE_TOKEN=tp_… python feed_reader.py
# Set THREADPAGE_FEED_ID to read a feed you already have; otherwise it creates one.
import json
import os
import pathlib
import time
import uuid

import requests

API = "https://thread.page/api/integrations/v1"
AUTH = {"Authorization": f"Bearer {os.environ['THREADPAGE_TOKEN']}"}
STATE = pathlib.Path(os.environ.get("THREADPAGE_FEED_STATE", "threadpage-feed.json"))


def load():
    return json.loads(STATE.read_text()) if STATE.exists() else {"seen": []}


def save(state):
    draft = STATE.with_suffix(".tmp")
    draft.write_text(json.dumps(state))
    draft.replace(STATE)  # a crash never leaves half a file


def create_feed(state):
    """POST /subscriptions with one Operation-Id, kept in the state file until the feed exists."""
    state.setdefault("create_operation_id", str(uuid.uuid4()))
    save(state)
    for attempt in range(1, 4):
        try:
            answer = requests.post(
                f"{API}/subscriptions",
                headers={**AUTH, "Operation-Id": state["create_operation_id"]},
                json={"kind": "pull", "event_types": ["note.created", "note.updated", "note.trashed"]},
                timeout=30,
            )
            break
        except (requests.ConnectionError, requests.Timeout):
            if attempt == 3:
                raise
            time.sleep(attempt * 2)
    answer.raise_for_status()
    state["feed_id"] = answer.json()["subscription"]["id"]
    del state["create_operation_id"]
    save(state)


def handle(event):
    data = event["data"]
    print(event["type"], (data.get("note") or data.get("thread") or {}).get("id"), flush=True)


def read_forever(state):
    while True:
        params = {"after": state["cursor"], "wait": 25} if state.get("cursor") else {}
        try:
            answer = requests.get(
                f"{API}/subscriptions/{state['feed_id']}/events", headers=AUTH, params=params, timeout=40
            )
        except (requests.ConnectionError, requests.Timeout):
            time.sleep(5)
            continue
        if answer.status_code == 410:  # the cursor is older than 7 days
            state["cursor"] = answer.json()["details"]["resync_cursor"]
            save(state)
            print("Events in between are not replayed: read current notes through the API.")
            continue
        if answer.status_code in (429, 503):
            time.sleep(int(answer.headers.get("Retry-After", 5)))
            continue
        if answer.status_code == 409:  # the feed is paused; its events wait
            time.sleep(60)
            continue
        answer.raise_for_status()
        page = answer.json()
        for event in page["events"]:
            if event["id"] in state["seen"]:
                continue  # the same event again
            handle(event)
            state["seen"] = [*state["seen"], event["id"]][-1000:]
            save(state)
        state["cursor"] = page["cursor"]  # only once the events are handled
        save(state)


if __name__ == "__main__":
    state = load()
    feed_id = os.environ.get("THREADPAGE_FEED_ID")
    if feed_id and state.get("feed_id") != feed_id:
        state = {"seen": [], "feed_id": feed_id}  # a cursor belongs to one feed
    if "feed_id" not in state:
        create_feed(state)
    read_forever(state)