Pagination & Conventions

async function listAllUsers() {
  const allUsers = [];
  let url = 'https://api.vomo.org/v1/users';

  while (url) {
    const response = await fetch(url, {
      headers: {
        Authorization: `Bearer ${process.env.VOMO_API_TOKEN}`,
        Accept: 'application/json',
      },
    });

    if (!response.ok) {
      throw new Error(`Page fetch failed: ${response.status}`);
    }

    const page = await response.json();
    allUsers.push(...page.data);

    // Follow the next link, or stop if there isn't one
    url = page.links.next;
  }

  return allUsers;
}

async function listUsersMatchingName(searchTerm) {
  const allMatches = [];
  const params = new URLSearchParams({ name_like: searchTerm });
  let url = `https://api.vomo.org/v1/users?${params}`;

  while (url) {
    const response = await fetch(url, {
      headers: { Authorization: `Bearer ${token}` },
    });
    const page = await response.json();
    allMatches.push(...page.data);
    url = page.links.next;
  }

  return allMatches;
}

async function findUserByEmail(email) {
  const params = new URLSearchParams({ email_like: email });
  let url = `https://api.vomo.org/v1/users?${params}`;

  while (url) {
    const response = await fetch(url, {
      headers: { Authorization: `Bearer ${token}` },
    });
    const page = await response.json();

    const match = page.data.find((u) => u.email?.toLowerCase() === email.toLowerCase());
    if (match) return match; // Found — stop reading

    url = page.links.next;
  }

  return null; // No match across all pages
}

async function backfillAllProjects(customerId, onProgress) {
  const allProjects = [];
  let url = 'https://api.vomo.org/v1/projects';
  let pagesRead = 0;
  let totalKnown = null;

  while (url) {
    const response = await fetch(url, {
      headers: { Authorization: `Bearer ${token}` },
    });
    const page = await response.json();

    allProjects.push(...page.data);
    pagesRead++;
    totalKnown ??= page.meta.total;

    if (onProgress) {
      onProgress({
        recordsLoaded: allProjects.length,
        totalRecords: totalKnown,
        currentPage: page.meta.current_page,
        totalPages: page.meta.last_page ?? Math.ceil(totalKnown / page.meta.per_page),
      });
    }

    url = page.links.next;
  }

  return allProjects;
}

// Usage
await backfillAllProjects(customerId, ({ recordsLoaded, totalRecords, currentPage }) => {
  console.log(`Page ${currentPage}: ${recordsLoaded}/${totalRecords} loaded`);
});

function formatPageRange(meta) {
  if (meta.total === 0) return 'No results';
  return `Showing ${meta.from}–${meta.to} of ${meta.total}`;
}

async function getPage(resource, pageNumber, filters = {}) {
  const params = new URLSearchParams({ ...filters, page: pageNumber.toString() });
  const url = `https://api.vomo.org/v1/${resource}?${params}`;

  const response = await fetch(url, {
    headers: { Authorization: `Bearer ${token}` },
  });

  return response.json();
}

// Get page 5 of users matching "wayne"
const page5 = await getPage('users', 5, { name_like: 'wayne' });

async function backfillResumable(customerId) {
  let currentPage = await getCheckpoint(customerId) ?? 1;

  while (true) {
    const params = new URLSearchParams({ page: currentPage.toString() });
    const response = await fetch(
      `https://api.vomo.org/v1/projects?${params}`,
      { headers: { Authorization: `Bearer ${token}` } }
    );
    const page = await response.json();

    if (page.data.length === 0) break;

    await processProjects(page.data);

    currentPage++;
    await setCheckpoint(customerId, currentPage);
  }
}

// ❌ Anti-pattern — fetches every page twice
const totalPages = await getTotalPages(); // 1 request
for (let i = 1; i <= totalPages; i++) {
  const page = await getPage(i); // N more requests
  await process(page);
}

// ✅ Use the cursor pattern — one pass through pages
let url = '/v1/users';
while (url) {
  const page = await fetch(url).then(r => r.json());
  await process(page);
  url = page.links.next;
}

// ❌ Anti-pattern — reads 1000 pages, filters to recent ones in code
const all = await listAllUsers();
const recent = all.filter((u) => new Date(u.created_at) > startOfYear);

// ✅ Push the filter into the request
const recentParams = new URLSearchParams({ created_after: startOfYear.toISOString() });
const recent = await listAllUsers(`?${recentParams}`);

// ❌ Memory-heavy — accumulates everything before processing
const all = await listAllUsers();
for (const user of all) {
  await processUser(user);
}

// ✅ Streaming — process each page as it arrives
async function streamAllUsers(processFn) {
  let url = 'https://api.vomo.org/v1/users';
  while (url) {
    const page = await fetch(url).then(r => r.json());
    for (const user of page.data) {
      await processFn(user);
    }
    url = page.links.next;
  }
}