'use strict';

const http = require('http');
const https = require('https');
const fs = require('fs');
const path = require('path');
const zlib = require('zlib');
const crypto = require('crypto');
const { URL } = require('url');

const HOST = '127.0.0.1';
const PORT = Number(process.env.PORT || 4173);
const ROOT = __dirname;
const CACHE_FILE = path.join(ROOT, 'catalog-cache.json');
const AGENT_STATE_FILE = process.env.MARKETING_BOSS_STATE_FILE || path.join(ROOT, 'marketing-boss-state.json');
const AGENT_OUTPUT_DIR = process.env.MARKETING_BOSS_OUTPUT_DIR || path.join(ROOT, 'marketing-output');
const CACHE_TTL_MS = 6 * 60 * 60 * 1000;
const USER_AGENT = 'SHIRE-Marketing-Forge-Easy/1.3 local-product-and-oauth-bridge';

const SOURCES = Object.freeze({
  shire3d_com: {
    id: 'shire3d_com', name: 'shire3d.com', kind: 'website', url: 'https://shire3d.com/',
    allowedHosts: ['shire3d.com', 'www.shire3d.com', 'shire3d.com.au', 'www.shire3d.com.au'], maxPages: 3, maxProducts: 180
  },
  shire3d_com_au: {
    id: 'shire3d_com_au', name: 'shire3d.com.au', kind: 'website', url: 'https://shire3d.com.au/s/shop',
    allowedHosts: ['shire3d.com.au', 'www.shire3d.com.au', 'shire3d.com', 'www.shire3d.com'], maxPages: 4, maxProducts: 180
  },
  shire3d_etsy: {
    id: 'shire3d_etsy', name: 'shire3d.etsy.com', kind: 'etsy', url: 'https://www.etsy.com/au/shop/Shire3D',
    allowedHosts: ['etsy.com', 'www.etsy.com'], maxPages: 5, maxProducts: 180
  }
});


const OAUTH_STATES = new Map();
const OAUTH_SESSIONS = new Map();
const OAUTH_STATE_TTL_MS = 10 * 60 * 1000;

function oauthProviders() {
  const base = `http://${HOST}:${PORT}`;
  return {
    facebook: {
      platform: 'facebook', label: 'Facebook', clientId: process.env.META_APP_ID || '', clientSecret: process.env.META_APP_SECRET || '',
      authUrl: 'https://www.facebook.com/dialog/oauth', tokenUrl: 'https://graph.facebook.com/oauth/access_token',
      scope: process.env.META_FACEBOOK_SCOPES || 'pages_show_list,pages_read_engagement,pages_manage_posts',
      redirectUri: process.env.FACEBOOK_REDIRECT_URI || `${base}/api/oauth/callback/facebook`,
      clientIdParam: 'client_id', developerPortal: 'https://developers.facebook.com/apps/',
      requiredEnvironment: ['META_APP_ID', 'META_APP_SECRET'],
      note: 'The Facebook user must manage the SHIRE3D Page. Meta may require App Review before Page publishing permissions work.'
    },
    instagram: {
      platform: 'instagram', label: 'Instagram', clientId: process.env.META_APP_ID || '', clientSecret: process.env.META_APP_SECRET || '',
      authUrl: 'https://www.facebook.com/dialog/oauth', tokenUrl: 'https://graph.facebook.com/oauth/access_token',
      scope: process.env.META_INSTAGRAM_SCOPES || 'instagram_basic,instagram_content_publish,pages_show_list,pages_read_engagement',
      redirectUri: process.env.INSTAGRAM_REDIRECT_URI || `${base}/api/oauth/callback/instagram`,
      clientIdParam: 'client_id', developerPortal: 'https://developers.facebook.com/apps/',
      requiredEnvironment: ['META_APP_ID', 'META_APP_SECRET'],
      note: 'Instagram publishing requires an eligible professional account and the permissions approved for the Meta app.'
    },
    linkedin: {
      platform: 'linkedin', label: 'LinkedIn', clientId: process.env.LINKEDIN_CLIENT_ID || '', clientSecret: process.env.LINKEDIN_CLIENT_SECRET || '',
      authUrl: 'https://www.linkedin.com/oauth/v2/authorization', tokenUrl: 'https://www.linkedin.com/oauth/v2/accessToken',
      scope: process.env.LINKEDIN_SCOPES || 'openid profile w_member_social',
      redirectUri: process.env.LINKEDIN_REDIRECT_URI || `${base}/api/oauth/callback/linkedin`,
      clientIdParam: 'client_id', developerPortal: 'https://www.linkedin.com/developers/apps',
      requiredEnvironment: ['LINKEDIN_CLIENT_ID', 'LINKEDIN_CLIENT_SECRET'],
      note: 'Company Page publishing can require additional LinkedIn Marketing API access and an approved Page administrator.'
    },
    tiktok: {
      platform: 'tiktok', label: 'TikTok', clientId: process.env.TIKTOK_CLIENT_KEY || '', clientSecret: process.env.TIKTOK_CLIENT_SECRET || '',
      authUrl: 'https://www.tiktok.com/v2/auth/authorize/', tokenUrl: 'https://open.tiktokapis.com/v2/oauth/token/',
      scope: process.env.TIKTOK_SCOPES || 'user.info.basic,video.upload,video.publish',
      redirectUri: process.env.TIKTOK_REDIRECT_URI || `${base}/api/oauth/callback/tiktok`,
      clientIdParam: 'client_key', developerPortal: 'https://developers.tiktok.com/apps/',
      requiredEnvironment: ['TIKTOK_CLIENT_KEY', 'TIKTOK_CLIENT_SECRET'], usesPkce: true,
      note: 'Use TikTok Desktop Login Kit and register the exact localhost callback. Direct Post access is reviewed separately by TikTok.'
    },
    youtube_shorts: {
      platform: 'youtube_shorts', label: 'YouTube Shorts', clientId: process.env.GOOGLE_CLIENT_ID || '', clientSecret: process.env.GOOGLE_CLIENT_SECRET || '',
      authUrl: 'https://accounts.google.com/o/oauth2/v2/auth', tokenUrl: 'https://oauth2.googleapis.com/token',
      scope: process.env.GOOGLE_YOUTUBE_SCOPES || 'openid profile https://www.googleapis.com/auth/youtube.upload',
      redirectUri: process.env.GOOGLE_REDIRECT_URI || `${base}/api/oauth/callback/youtube_shorts`,
      clientIdParam: 'client_id', developerPortal: 'https://console.cloud.google.com/apis/credentials',
      requiredEnvironment: ['GOOGLE_CLIENT_ID', 'GOOGLE_CLIENT_SECRET'],
      authExtras: { access_type: 'offline', include_granted_scopes: 'true', prompt: 'consent' },
      note: 'Enable the YouTube Data API in the Google Cloud project. Shorts are uploaded through the normal YouTube video upload API.'
    }
  };
}

function randomUrlSafe(bytes = 32) {
  return crypto.randomBytes(bytes).toString('base64url');
}

function tiktokCodeChallenge(verifier) {
  return crypto.createHash('sha256').update(verifier).digest('hex');
}

function decodeJwtPayload(token) {
  try {
    const part = String(token || '').split('.')[1];
    if (!part) return {};
    return JSON.parse(Buffer.from(part, 'base64url').toString('utf8'));
  } catch (_) { return {}; }
}

function oauthPublicStatus(provider) {
  const session = OAUTH_SESSIONS.get(provider.platform);
  return {
    platform: provider.platform,
    label: provider.label,
    configured: Boolean(provider.clientId && provider.clientSecret),
    connected: Boolean(session && session.accessToken),
    status: session && session.accessToken ? 'connected' : (provider.clientId && provider.clientSecret ? 'ready_to_login' : 'setup_required'),
    accountName: session ? session.accountName : '',
    connectedAt: session ? session.connectedAt : null,
    expiresAt: session ? session.expiresAt : null,
    redirectUri: provider.redirectUri,
    developerPortal: provider.developerPortal,
    requiredEnvironment: provider.requiredEnvironment,
    note: provider.note,
    tokenStorage: 'memory_only'
  };
}

function cleanupOauthStates() {
  const cutoff = Date.now() - OAUTH_STATE_TTL_MS;
  for (const [key, value] of OAUTH_STATES.entries()) {
    if (!value || value.createdAt < cutoff) OAUTH_STATES.delete(key);
  }
}

function redirectToForge(res, result, platform, message) {
  const params = new URLSearchParams({ oauth: result, platform: platform || '' });
  if (message) params.set('message', String(message).slice(0, 300));
  res.writeHead(302, { Location: `/?${params.toString()}#accounts`, 'Cache-Control': 'no-store' });
  res.end();
}

async function exchangeOauthCode(provider, code, verifier) {
  const body = new URLSearchParams();
  body.set(provider.clientIdParam, provider.clientId);
  body.set('client_secret', provider.clientSecret);
  body.set('code', code);
  body.set('grant_type', 'authorization_code');
  body.set('redirect_uri', provider.redirectUri);
  if (provider.usesPkce && verifier) body.set('code_verifier', verifier);
  const response = await fetch(provider.tokenUrl, {
    method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded', 'Accept': 'application/json', 'User-Agent': USER_AGENT },
    body
  });
  const raw = await response.text();
  let payload;
  try { payload = JSON.parse(raw); } catch (_) { payload = Object.fromEntries(new URLSearchParams(raw)); }
  if (!response.ok || !payload.access_token) {
    const detail = payload.error_description || (payload.error && payload.error.message) || payload.error || `OAuth token request returned HTTP ${response.status}.`;
    throw new Error(String(detail));
  }
  return payload;
}

function accountNameFromToken(provider, payload) {
  const claims = decodeJwtPayload(payload.id_token);
  return String(claims.name || claims.email || payload.display_name || `SHIRE3D ${provider.label}`).slice(0, 180);
}

const MIME = {
  '.html': 'text/html; charset=utf-8', '.js': 'text/javascript; charset=utf-8', '.css': 'text/css; charset=utf-8',
  '.json': 'application/json; charset=utf-8', '.svg': 'image/svg+xml', '.png': 'image/png', '.jpg': 'image/jpeg',
  '.jpeg': 'image/jpeg', '.webp': 'image/webp', '.md': 'text/markdown; charset=utf-8', '.webmanifest': 'application/manifest+json; charset=utf-8'
};

function sendJson(res, status, payload) {
  const body = JSON.stringify(payload, null, 2);
  res.writeHead(status, {
    'Content-Type': 'application/json; charset=utf-8', 'Content-Length': Buffer.byteLength(body),
    'Cache-Control': 'no-store', 'X-Content-Type-Options': 'nosniff'
  });
  res.end(body);
}

function normalizeHost(value) { return String(value || '').toLowerCase().replace(/^www\./, ''); }
function hostAllowed(source, hostname) { return source.allowedHosts.some((x) => normalizeHost(x) === normalizeHost(hostname)); }

function fetchText(urlString, redirects = 0) {
  return new Promise((resolve, reject) => {
    if (redirects > 6) return reject(new Error('Too many redirects.'));
    let parsed;
    try { parsed = new URL(urlString); } catch (_) { return reject(new Error('Invalid URL.')); }
    if (!['https:', 'http:'].includes(parsed.protocol)) return reject(new Error('Only HTTP(S) is allowed.'));
    const client = parsed.protocol === 'https:' ? https : http;
    const req = client.request(parsed, {
      method: 'GET', timeout: 12000,
      headers: {
        'User-Agent': USER_AGENT,
        'Accept': 'text/html,application/xhtml+xml,application/json;q=0.9,*/*;q=0.7',
        'Accept-Language': 'en-AU,en;q=0.9', 'Accept-Encoding': 'gzip, deflate, br'
      }
    }, (response) => {
      const status = response.statusCode || 0;
      if ([301, 302, 303, 307, 308].includes(status) && response.headers.location) {
        const next = new URL(response.headers.location, parsed).href;
        response.resume();
        return resolve(fetchText(next, redirects + 1));
      }
      if (status < 200 || status >= 300) {
        response.resume();
        return reject(new Error(`Website returned HTTP ${status}.`));
      }
      const chunks = [];
      let size = 0;
      response.on('data', (chunk) => {
        size += chunk.length;
        if (size > 10 * 1024 * 1024) req.destroy(new Error('Website response exceeded the safe size limit.'));
        else chunks.push(chunk);
      });
      response.on('end', () => {
        try {
          let buffer = Buffer.concat(chunks);
          const encoding = String(response.headers['content-encoding'] || '').toLowerCase();
          if (encoding.includes('gzip')) buffer = zlib.gunzipSync(buffer);
          else if (encoding.includes('deflate')) buffer = zlib.inflateSync(buffer);
          else if (encoding.includes('br')) buffer = zlib.brotliDecompressSync(buffer);
          resolve({ url: parsed.href, body: buffer.toString('utf8') });
        } catch (error) { reject(error); }
      });
    });
    req.on('timeout', () => req.destroy(new Error('Website request timed out.')));
    req.on('error', reject);
    req.end();
  });
}

function decodeEntities(value) {
  return String(value || '')
    .replace(/&amp;/gi, '&').replace(/&quot;/gi, '"').replace(/&#39;|&apos;/gi, "'")
    .replace(/&lt;/gi, '<').replace(/&gt;/gi, '>')
    .replace(/&#x([0-9a-f]+);/gi, (_, x) => String.fromCodePoint(parseInt(x, 16)))
    .replace(/&#(\d+);/g, (_, x) => String.fromCodePoint(Number(x))).replace(/\s+/g, ' ').trim();
}
function stripTags(value) {
  return decodeEntities(String(value || '').replace(/<script\b[^>]*>[\s\S]*?<\/script>/gi, ' ')
    .replace(/<style\b[^>]*>[\s\S]*?<\/style>/gi, ' ').replace(/<[^>]+>/g, ' '));
}
function attrs(tag) {
  const result = {};
  String(tag || '').replace(/([:\w-]+)\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+))/g, (_, key, a, b, c) => {
    result[key.toLowerCase()] = decodeEntities(a || b || c || ''); return '';
  });
  return result;
}
function meta(html, names) {
  const wanted = new Set(names.map((x) => x.toLowerCase()));
  for (const tag of String(html || '').match(/<meta\b[^>]*>/gi) || []) {
    const a = attrs(tag); const key = String(a.property || a.name || a.itemprop || '').toLowerCase();
    if (wanted.has(key) && a.content) return a.content;
  }
  return '';
}
function price(value) {
  if (typeof value === 'number') return Number.isFinite(value) ? value : 0;
  if (typeof value === 'string') { const n = Number(value.replace(/[^0-9.-]/g, '')); return Number.isFinite(n) ? n : 0; }
  if (Array.isArray(value)) { for (const x of value) { const n = price(x); if (n > 0) return n; } }
  if (value && typeof value === 'object') return price(value.price || value.lowPrice || value.highPrice || value.amount);
  return 0;
}
function image(value) {
  if (!value) return ''; if (typeof value === 'string') return value; if (Array.isArray(value)) return image(value[0]);
  if (typeof value === 'object') return value.url || value.contentUrl || ''; return '';
}
function cleanTitle(value) {
  return stripTags(value).replace(/^Loading\s+/i, '').replace(/\s*[|–—-]\s*(Shire\s*3D|Etsy.*)$/i, '').trim();
}
function productFromLd(node, pageUrl) {
  if (!node || typeof node !== 'object') return null;
  const type = Array.isArray(node['@type']) ? node['@type'].join(' ') : String(node['@type'] || '');
  if (!/\bProduct\b/i.test(type)) return null;
  const offers = node.offers || {};
  const name = cleanTitle(node.name || node.headline || '');
  if (!name) return null;
  let listingUrl = node.url || node['@id'] || offers.url || pageUrl;
  try { listingUrl = new URL(listingUrl, pageUrl).href; } catch (_) { listingUrl = pageUrl; }
  const availability = String(offers.availability || '').toLowerCase();
  return {
    name,
    description: stripTags(node.description || '').slice(0, 4000),
    price: price(offers),
    currency: String(offers.priceCurrency || 'AUD').toUpperCase(),
    imageUrl: image(node.image), listingUrl,
    externalId: String(node.sku || node.productID || node.mpn || ''),
    stockStatus: /outofstock|soldout|unavailable/.test(availability) ? 'out of stock' : 'available'
  };
}

function collectJsonLdProducts(html, pageUrl) {
  const found = [];
  const scripts = String(html || '').match(/<script\b[^>]*type\s*=\s*["']application\/ld\+json["'][^>]*>[\s\S]*?<\/script>/gi) || [];
  for (const script of scripts) {
    const raw = script.replace(/^<script\b[^>]*>/i, '').replace(/<\/script>$/i, '').trim();
    try {
      const parsed = JSON.parse(raw);
      const walk = (node) => {
        if (!node) return;
        if (Array.isArray(node)) return node.forEach(walk);
        if (typeof node !== 'object') return;
        const product = productFromLd(node, pageUrl);
        if (product) found.push(product);
        Object.keys(node).forEach((key) => { if (!['offers', 'brand', 'image'].includes(key)) walk(node[key]); });
      };
      walk(parsed);
    } catch (_) {}
  }
  return found;
}

function extractAnchorProducts(html, baseUrl, source) {
  const found = [];
  const sourceHtml = String(html || '');
  const regex = /<a\b([^>]*)href\s*=\s*(?:"([^"]+)"|'([^']+)')([^>]*)>([\s\S]*?)<\/a>/gi;
  let match;
  while ((match = regex.exec(sourceHtml))) {
    let url;
    try { url = new URL(decodeEntities(match[2] || match[3] || ''), baseUrl); } catch (_) { continue; }
    if (!hostAllowed(source, url.hostname)) continue;
    const isProduct = source.kind === 'etsy' ? /\/listing\/\d+/i.test(url.pathname) : /\/product\//i.test(url.pathname);
    if (!isProduct) continue;
    url.hash = '';
    const anchorAttrs = attrs(`<a ${match[1] || ''} ${match[4] || ''}>`);
    const inner = match[5] || '';
    const around = sourceHtml.slice(Math.max(0, match.index - 700), Math.min(sourceHtml.length, regex.lastIndex + 1200));
    let name = cleanTitle(anchorAttrs['aria-label'] || anchorAttrs.title || stripTags(inner));
    if (!name || name.length > 220 || /add to|favourite|image$/i.test(name)) {
      const imageTag = inner.match(/<img\b[^>]*>/i) || around.match(/<img\b[^>]*>/i);
      const imageAttrs = imageTag ? attrs(imageTag[0]) : {};
      name = cleanTitle(imageAttrs.alt || name);
    }
    const plain = stripTags(around);
    const amount = (plain.match(/(?:A\$|AU\$|AUD\s*\$?|\$)\s*([0-9]+(?:[.,][0-9]{1,2})?)/i) || [])[1] || '';
    const imageTag = inner.match(/<img\b[^>]*>/i) || around.match(/<img\b[^>]*>/i);
    const imageAttrs = imageTag ? attrs(imageTag[0]) : {};
    let imageUrl = imageAttrs.src || imageAttrs['data-src'] || imageAttrs['data-lazy-src'] || '';
    try { if (imageUrl) imageUrl = new URL(imageUrl, baseUrl).href; } catch (_) { imageUrl = ''; }
    const idMatch = source.kind === 'etsy' ? url.pathname.match(/\/listing\/(\d+)/i) : url.pathname.match(/\/product\/[^/]+\/([^/?]+)/i);
    found.push({
      name, description: '', price: amount ? Number(amount.replace(',', '.')) : 0, currency: 'AUD', imageUrl,
      listingUrl: url.href, externalId: idMatch ? idMatch[1] : '', stockStatus: /sold out|out of stock/i.test(plain) ? 'out of stock' : 'available'
    });
  }
  return found;
}

function parseSingleProductPage(html, pageUrl, source) {
  const ld = collectJsonLdProducts(html, pageUrl);
  if (ld.length) return ld[0];
  const titleTag = String(html || '').match(/<title\b[^>]*>([\s\S]*?)<\/title>/i);
  const name = cleanTitle(meta(html, ['og:title', 'twitter:title']) || (titleTag && titleTag[1]) || '');
  if (!name) return null;
  const listingUrl = meta(html, ['og:url']) || pageUrl;
  let parsed;
  try { parsed = new URL(listingUrl, pageUrl); } catch (_) { parsed = new URL(pageUrl); }
  const idMatch = source.kind === 'etsy' ? parsed.pathname.match(/\/listing\/(\d+)/i) : parsed.pathname.match(/\/product\/[^/]+\/([^/?]+)/i);
  const body = stripTags(html).slice(0, 200000);
  const rawPrice = meta(html, ['product:price:amount', 'og:price:amount', 'price']) || ((body.match(/(?:A\$|AU\$|AUD\s*\$?|\$)\s*([0-9]+(?:[.,][0-9]{1,2})?)/i) || [])[1] || '');
  return {
    name, description: stripTags(meta(html, ['og:description', 'description', 'twitter:description'])).slice(0, 4000),
    price: price(rawPrice), currency: String(meta(html, ['product:price:currency', 'og:price:currency']) || 'AUD').toUpperCase(),
    imageUrl: meta(html, ['og:image', 'twitter:image']), listingUrl: parsed.href,
    externalId: idMatch ? idMatch[1] : '', stockStatus: /sold out|out of stock|unavailable/i.test(body) ? 'out of stock' : 'available'
  };
}

function productKey(product) {
  if (product.externalId) return `id:${String(product.externalId).toLowerCase()}`;
  if (product.listingUrl) {
    try { const u = new URL(product.listingUrl); u.search = ''; u.hash = ''; return `url:${u.href.toLowerCase()}`; } catch (_) {}
  }
  return `name:${String(product.name || '').toLowerCase().replace(/[^a-z0-9]+/g, '')}`;
}

function mergeProducts(records) {
  const map = new Map();
  records.forEach((raw) => {
    const item = {
      ...raw, name: cleanTitle(raw.name), description: stripTags(raw.description || '').slice(0, 4000), price: price(raw.price),
      currency: String(raw.currency || 'AUD').toUpperCase().slice(0, 8), imageUrl: String(raw.imageUrl || '').slice(0, 2000),
      listingUrl: String(raw.listingUrl || '').slice(0, 2000), externalId: String(raw.externalId || '').slice(0, 160),
      stockStatus: String(raw.stockStatus || 'available').slice(0, 80)
    };
    if (!item.name || item.name.length < 3) return;
    const key = productKey(item); const old = map.get(key);
    map.set(key, old ? {
      ...old, ...item, name: item.name || old.name, description: item.description || old.description,
      price: item.price || old.price, imageUrl: item.imageUrl || old.imageUrl, externalId: item.externalId || old.externalId
    } : item);
  });
  return Array.from(map.values());
}

async function concurrent(items, limit, worker) {
  const output = new Array(items.length); let cursor = 0;
  const runners = Array.from({ length: Math.min(limit, items.length) }, async () => {
    while (true) {
      const index = cursor++; if (index >= items.length) break;
      try { output[index] = await worker(items[index], index); } catch (error) { output[index] = { error }; }
    }
  });
  await Promise.all(runners); return output;
}
function pageUrls(source) {
  const urls = [source.url];
  if (source.kind === 'etsy') {
    for (let page = 2; page <= source.maxPages; page += 1) {
      const url = new URL(source.url); url.searchParams.set('page', String(page)); urls.push(url.href);
    }
  } else {
    const origin = new URL(source.url).origin;
    ['/s/shop', '/shop'].forEach((pathname) => { const url = new URL(pathname, origin).href; if (!urls.includes(url)) urls.push(url); });
  }
  return urls.slice(0, source.maxPages);
}

async function syncSource(source) {
  const startedAt = new Date().toISOString();
  const pages = await concurrent(pageUrls(source), 3, async (url) => {
    const response = await fetchText(url);
    const finalUrl = new URL(response.url);
    if (!hostAllowed(source, finalUrl.hostname)) throw new Error('Redirected outside the approved SHIRE3D source boundary.');
    return response;
  });
  const goodPages = pages.filter((x) => x && !x.error);
  if (!goodPages.length) {
    const firstError = pages.find((x) => x && x.error);
    throw firstError ? firstError.error : new Error('No source pages could be read.');
  }
  let products = [];
  goodPages.forEach((page) => {
    products.push(...collectJsonLdProducts(page.body, page.url));
    products.push(...extractAnchorProducts(page.body, page.url, source));
  });
  products = mergeProducts(products);

  const incomplete = products.filter((x) => !x.description || !x.price || !x.imageUrl).slice(0, 12);
  if (incomplete.length) {
    const details = await concurrent(incomplete, 6, async (item) => {
      const url = new URL(item.listingUrl);
      if (!hostAllowed(source, url.hostname)) throw new Error('Product link left the approved source boundary.');
      const response = await fetchText(url.href);
      return parseSingleProductPage(response.body, response.url, source);
    });
    details.forEach((detail, index) => {
      if (!detail || detail.error) return;
      const target = productKey(incomplete[index]);
      products = products.map((item) => productKey(item) === target ? {
        ...item, ...detail, name: detail.name || item.name, description: detail.description || item.description,
        price: detail.price || item.price, imageUrl: detail.imageUrl || item.imageUrl
      } : item);
    });
  }

  products = mergeProducts(products).slice(0, source.maxProducts).map((item) => ({
    ...item, sourceId: source.id, sourceName: source.name, sourceUrl: source.url,
    syncedAt: new Date().toISOString(), trustedSource: true
  }));
  return {
    source: { id: source.id, name: source.name, url: source.url, kind: source.kind },
    startedAt, completedAt: new Date().toISOString(), count: products.length, products,
    warnings: products.length ? [] : ['The storefront responded, but no product listings were found. Its page layout may have changed.']
  };
}

function loadCache() {
  try { const value = JSON.parse(fs.readFileSync(CACHE_FILE, 'utf8')); return value && typeof value === 'object' ? value : {}; }
  catch (_) { return {}; }
}
function saveCache(value) { try { fs.writeFileSync(CACHE_FILE, JSON.stringify(value, null, 2), 'utf8'); } catch (_) {} }
function loadAgentState() {
  try { const value = JSON.parse(fs.readFileSync(AGENT_STATE_FILE, 'utf8')); return value && typeof value === 'object' ? value : null; }
  catch (_) { return null; }
}
function saveAgentState(value) {
  const parent = path.dirname(AGENT_STATE_FILE);
  fs.mkdirSync(parent, { recursive: true });
  const temp = `${AGENT_STATE_FILE}.tmp`;
  fs.writeFileSync(temp, JSON.stringify(value, null, 2), { encoding: 'utf8', mode: 0o600 });
  fs.renameSync(temp, AGENT_STATE_FILE);
}
async function getSourceResult(source, force) {
  const cache = loadCache(); const current = cache[source.id];
  if (!force && current && current.cachedAt && Date.now() - new Date(current.cachedAt).getTime() < CACHE_TTL_MS && current.result) {
    return { ...current.result, fromCache: true };
  }
  const result = await syncSource(source);
  cache[source.id] = { cachedAt: new Date().toISOString(), result }; saveCache(cache);
  return { ...result, fromCache: false };
}

async function readBody(req) {
  const chunks = []; let size = 0;
  for await (const chunk of req) {
    size += chunk.length; if (size > 5 * 1024 * 1024) throw new Error('Request body too large.'); chunks.push(chunk);
  }
  return chunks.length ? JSON.parse(Buffer.concat(chunks).toString('utf8')) : {};
}

function serveStatic(req, res) {
  let pathname;
  try { pathname = decodeURIComponent(new URL(req.url, `http://${HOST}:${PORT}`).pathname); }
  catch (_) { return sendJson(res, 400, { error: 'Bad request.' }); }
  if (pathname === '/') pathname = '/index.html';
  const file = path.resolve(ROOT, `.${pathname}`); const root = path.resolve(ROOT);
  if (file !== path.join(root, 'index.html') && !file.startsWith(`${root}${path.sep}`)) return sendJson(res, 403, { error: 'Forbidden.' });
  fs.stat(file, (error, stat) => {
    if (error || !stat.isFile()) return sendJson(res, 404, { error: 'Not found.' });
    const ext = path.extname(file).toLowerCase();
    res.writeHead(200, {
      'Content-Type': MIME[ext] || 'application/octet-stream',
      'Cache-Control': ext === '.html' ? 'no-cache' : 'public, max-age=300', 'X-Content-Type-Options': 'nosniff'
    });
    fs.createReadStream(file).pipe(res);
  });
}

const server = http.createServer(async (req, res) => {
  const url = new URL(req.url, `http://${HOST}:${PORT}`);
  if (url.pathname === '/agent-review') {
    const file = path.join(AGENT_OUTPUT_DIR, 'latest-campaign-review.html');
    try {
      const body = fs.readFileSync(file);
      res.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8', 'Content-Length': body.length, 'Cache-Control': 'no-store', 'X-Content-Type-Options': 'nosniff' });
      return res.end(body);
    } catch (_) { return sendJson(res, 404, { error: 'No Marketing Boss campaign review is available yet.' }); }
  }

  if (url.pathname === '/api/health') {
    return sendJson(res, 200, {
      ok: true, app: 'SHIRE Marketing Forge Easy', version: '1.3.0', boundary: 'SHIRE3D_ONLY',
      sources: Object.values(SOURCES).map(({ id, name, url, kind }) => ({ id, name, url, kind }))
    });
  }
  if (url.pathname === '/api/agent-state') {
    if (req.method === 'GET') return sendJson(res, 200, { ok: true, boundary: 'SHIRE3D_ONLY', state: loadAgentState() });
    if (req.method === 'POST') {
      try {
        const body = await readBody(req);
        const value = body && body.state ? body.state : body;
        if (!value || typeof value !== 'object' || Array.isArray(value)) return sendJson(res, 400, { error: 'A state object is required.' });
        const text = JSON.stringify(value);
        if (/kitchen\s*hero/i.test(text)) return sendJson(res, 400, { error: 'Kitchen Hero separation boundary blocked this state.' });
        saveAgentState(value);
        return sendJson(res, 200, { ok: true, boundary: 'SHIRE3D_ONLY' });
      } catch (error) { return sendJson(res, 400, { error: error.message }); }
    }
    return sendJson(res, 405, { error: 'Method not allowed.' });
  }

  if (url.pathname === '/api/oauth/status') {
    const providers = Object.values(oauthProviders()).map(oauthPublicStatus);
    return sendJson(res, 200, { ok: true, boundary: 'SHIRE3D_ONLY', storage: 'server_memory_only', providers });
  }

  const oauthStart = url.pathname.match(/^\/api\/oauth\/start\/([a-z_]+)$/);
  if (oauthStart) {
    const platform = oauthStart[1];
    const provider = oauthProviders()[platform];
    if (!provider) return redirectToForge(res, 'error', platform, 'Unsupported social platform.');
    if (!provider.clientId || !provider.clientSecret) return redirectToForge(res, 'error', platform, 'The one-time developer app setup has not been completed.');
    cleanupOauthStates();
    const stateToken = randomUrlSafe(28);
    const verifier = provider.usesPkce ? randomUrlSafe(48) : '';
    OAUTH_STATES.set(stateToken, { platform, verifier, createdAt: Date.now() });
    const auth = new URL(provider.authUrl);
    auth.searchParams.set(provider.clientIdParam, provider.clientId);
    auth.searchParams.set('redirect_uri', provider.redirectUri);
    auth.searchParams.set('response_type', 'code');
    auth.searchParams.set('scope', provider.scope);
    auth.searchParams.set('state', stateToken);
    if (provider.usesPkce) {
      auth.searchParams.set('code_challenge', tiktokCodeChallenge(verifier));
      auth.searchParams.set('code_challenge_method', 'S256');
    }
    Object.entries(provider.authExtras || {}).forEach(([key, value]) => auth.searchParams.set(key, value));
    res.writeHead(302, { Location: auth.href, 'Cache-Control': 'no-store' });
    return res.end();
  }

  const oauthCallback = url.pathname.match(/^\/api\/oauth\/callback\/([a-z_]+)$/);
  if (oauthCallback) {
    const platform = oauthCallback[1];
    const provider = oauthProviders()[platform];
    if (!provider) return redirectToForge(res, 'error', platform, 'Unsupported social platform.');
    const error = url.searchParams.get('error');
    if (error) return redirectToForge(res, 'error', platform, url.searchParams.get('error_description') || error);
    const stateToken = url.searchParams.get('state') || '';
    const pending = OAUTH_STATES.get(stateToken);
    OAUTH_STATES.delete(stateToken);
    if (!pending || pending.platform !== platform || Date.now() - pending.createdAt > OAUTH_STATE_TTL_MS) {
      return redirectToForge(res, 'error', platform, 'The login request expired or failed its security check. Please try again.');
    }
    const code = url.searchParams.get('code');
    if (!code) return redirectToForge(res, 'error', platform, 'No authorization code was returned.');
    try {
      const payload = await exchangeOauthCode(provider, code, pending.verifier);
      const expiresIn = Number(payload.expires_in || 0);
      OAUTH_SESSIONS.set(platform, {
        platform,
        accessToken: payload.access_token,
        refreshToken: payload.refresh_token || '',
        tokenType: payload.token_type || 'Bearer',
        scope: payload.scope || provider.scope,
        accountName: accountNameFromToken(provider, payload),
        connectedAt: new Date().toISOString(),
        expiresAt: expiresIn > 0 ? new Date(Date.now() + expiresIn * 1000).toISOString() : null
      });
      return redirectToForge(res, 'connected', platform, '');
    } catch (oauthError) {
      return redirectToForge(res, 'error', platform, oauthError.message);
    }
  }

  const oauthDisconnect = url.pathname.match(/^\/api\/oauth\/disconnect\/([a-z_]+)$/);
  if (oauthDisconnect) {
    if (req.method !== 'POST') return sendJson(res, 405, { error: 'Method not allowed.' });
    const platform = oauthDisconnect[1];
    if (!oauthProviders()[platform]) return sendJson(res, 404, { error: 'Unsupported social platform.' });
    OAUTH_SESSIONS.delete(platform);
    return sendJson(res, 200, { ok: true, platform, disconnected: true });
  }

  if (url.pathname === '/api/sync-products') {
    if (!['GET', 'POST'].includes(req.method)) return sendJson(res, 405, { error: 'Method not allowed.' });
    try {
      const body = req.method === 'POST' ? await readBody(req) : {};
      const requested = Array.isArray(body.sourceIds) && body.sourceIds.length ? body.sourceIds : Object.keys(SOURCES);
      const ids = requested.filter((id) => Object.prototype.hasOwnProperty.call(SOURCES, id));
      if (!ids.length) return sendJson(res, 400, { error: 'No approved SHIRE3D sources selected.' });
      const force = Boolean(body.force);
      const cache = loadCache();
      const results = await Promise.all(ids.map(async (id) => {
        const source = SOURCES[id];
        const current = cache[id];
        if (!force && current && current.cachedAt && Date.now() - new Date(current.cachedAt).getTime() < CACHE_TTL_MS && current.result) {
          return { ...current.result, fromCache: true };
        }
        try {
          const result = await syncSource(source);
          cache[id] = { cachedAt: new Date().toISOString(), result };
          return { ...result, fromCache: false };
        } catch (error) {
          return { source: { id, name: source.name, url: source.url, kind: source.kind }, completedAt: new Date().toISOString(), count: 0, products: [], error: error.message };
        }
      }));
      saveCache(cache);
      return sendJson(res, 200, { ok: results.some((x) => !x.error), boundary: 'SHIRE3D_ONLY', results });
    } catch (error) { return sendJson(res, 400, { error: error.message }); }
  }
  return serveStatic(req, res);
});

if (require.main === module) {
  server.listen(PORT, HOST, () => console.log(`SHIRE Marketing Forge Easy v1.3.0 running at http://${HOST}:${PORT}`));
  server.on('error', (error) => {
    if (error.code === 'EADDRINUSE') console.error(`Port ${PORT} is already in use. The Forge may already be running.`);
    else console.error(error);
    process.exitCode = 1;
  });
}

module.exports = { SOURCES, oauthProviders, oauthPublicStatus, tiktokCodeChallenge, decodeEntities, stripTags, collectJsonLdProducts, extractAnchorProducts, parseSingleProductPage, mergeProducts, productKey };
