Function bodies 406 total

async function audit() {
  console.log('\n╔════════════════════════════════════════════════════════════════════╗');
  console.log('║              LEMLIST CAMPAIGN AUDIT                               ║');
  console.log('╚════════════════════════════════════════════════════════════════════╝\n');

  console.log(`Campaign: ${CAMPAIGN_ID}`);
  console.log(`Mode: ${VERIFY_MODE ? 'AUDIT + VERIFY + REMOVE INVALID' : 'AUDIT ONLY (read-only)'}\n`);

  // Step 1: Pull all leads
  console.log('Fetching leads from Lemlist...\n');
  let leads;
  try {
    leads = await getLeadsFromCampaign(CAMPAIGN_ID);
  } catch (err) {
    console.error(`Failed to fetch leads: ${err.message}`);
    process.exit(1);
  }

  if (!leads || leads.length === 0) {
    console.log('No leads found in campaign.\n');
    process.exit(0);
  }

  console.log(`Found ${leads.length} leads\n`);

  // Step 2: Audit each lead
  const issues = [];    // { lead, problems: string[] }
  const byCategory = {};
  const byNameQuality = { 

function pct(n, total) {
  if (total === 0) return '0%';
  return `${Math.round(n / total * 100)}%`;
}

function makeRequest(path) {
  return new Promise((resolve, reject) => {
    const apiKey = getCredential('lemlist', 'apiKey');
    const auth = Buffer.from(`:${apiKey}`).toString('base64');

    const options = {
      hostname: 'api.lemlist.com',
      path: path,
      method: 'GET',
      headers: {
        'Authorization': `Basic ${auth}`,
        'Content-Type': 'application/json'
      }
    };

    const req = https.request(options, (res) => {
      let data = '';
      res.on('data', chunk => data += chunk);
      res.on('end', () => {
        if (res.statusCode >= 200 && res.statusCode < 300) {
          try {
            resolve(JSON.parse(data));
          } catch (e) {
            resolve(data);
          }
        } else {
          reject(new Error(`HTTP ${res.statusCode}: ${data}`));
        }
      });
    });

    req.on('error', reject);
    req.end();
  });
}

async function checkLemlistStatus() {
  console.log('\n╔════════════════════════════════════════════════════════════════════╗');
  console.log('║              LEMLIST CAMPAIGN STATUS CHECK                         ║');
  console.log('╚════════════════════════════════════════════════════════════════════╝\n');

  console.log(`Campaign ID: ${CAMPAIGN_ID}\n`);

  try {
    // Get all leads
    console.log('Fetching leads from Lemlist...\n');
    const leads = await makeRequest(`/api/campaigns/${CAMPAIGN_ID}/leads`);

    console.log(`━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━`);
    console.log(`TOTAL LEADS: ${leads.length}`);
    console.log(`━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n`);

    // Analyze leads
    let verified = 0;
    let notVerified = 0;
    let contacted = 0;
    let notContacted = 0;
    let replied = 0;
    let bounced = 0;

    const emailDomains = {};
    const companyNames = [];
    const missingData = {
      noPhon

function isBadName(firstName, lastName) {
  const first = (firstName || '').toLowerCase().trim();
  const last = (lastName || '').toLowerCase().trim();

  // Already fallback or empty - skip
  if (first === 'there' || first === '') return false;

  // Check bad first names
  if (BAD_FIRST_NAMES.has(first)) return true;

  // Check bad last names
  if (BAD_LAST_NAMES.has(last)) return true;

  // "Su" as last name with any first name
  if (last === 'su') return true;

  // Last name is 2 chars or less and not a known short surname
  if (last.length > 0 && last.length <= 2 && !isCommonShortSurname(last)) return true;

  // First name looks like a company/business word
  if (isBusinessWord(first)) return true;

  // Last name looks like a business descriptor
  if (isBusinessWord(last)) return true;

  return false;
}

function isCommonShortSurname(name) {
  const realShortSurnames = new Set([
    'li', 'wu', 'xu', 'ye', 'ma', 'he', 'hu', 'lu', 'ng',
    'ho', 'lo', 'ko', 'do', 'le', 'ly', 'qi', 'yu', 'ai'
  ]);
  return realShortSurnames.has(name.toLowerCase());
}

function isBusinessWord(word) {
  const businessWords = new Set([
    'accountancy', 'accounting', 'insurance', 'structural', 'consulting',
    'commercial', 'community', 'engineering', 'recruitment', 'chartered',
    'management', 'certified', 'protection', 'employment', 'elimination',
    'professional', 'construction', 'architectural', 'financial',
    'digital', 'response', 'approaches', 'solutions', 'services',
    'associates', 'partnership', 'enterprises', 'holdings', 'group',
    'limited', 'ltd', 'plc', 'inc', 'corp',
    'diet', 'files', 'operators', 'businesses', 'survey', 'socials',
    'recurring', 'mixed', 'cutter', 'attach',
  ]);
  return businessWords.has(word.toLowerCase());
}

function isBadEmail(email) {
  if (!email) return true;

  // Check bad domains
  for (const domain of BAD_EMAIL_DOMAINS) {
    if (email.endsWith('@' + domain)) return true;
  }

  // Check bad patterns (hex IDs)
  for (const pattern of BAD_EMAIL_PATTERNS) {
    if (pattern.test(email)) return true;
  }

  return false;
}

function makeRequest(path, method, body) {
  return new Promise((resolve, reject) => {
    const apiKey = getCredential('lemlist', 'apiKey');
    const auth = Buffer.from(`:${apiKey}`).toString('base64');

    const options = {
      hostname: 'api.lemlist.com',
      path: path,
      method: method || 'GET',
      headers: {
        'Authorization': `Basic ${auth}`,
        'Content-Type': 'application/json'
      }
    };

    if (body) {
      const postData = JSON.stringify(body);
      options.headers['Content-Length'] = Buffer.byteLength(postData);
    }

    const req = https.request(options, (res) => {
      const chunks = [];
      res.on('data', chunk => chunks.push(chunk));
      res.on('end', () => {
        const data = Buffer.concat(chunks).toString('utf8');
        if (res.statusCode >= 200 && res.statusCode < 300) {
          try {
            resolve(data ? JSON.parse(data) : { success: true });
          } catch (e) {
            resolve(data || { success: true });
 

async function updateLeadInLemlist(email, updates) {
  return makeRequest(
    `/api/campaigns/${CAMPAIGN_ID}/leads/${encodeURIComponent(email)}`,
    'PATCH',
    updates
  );
}

async function deleteLeadFromLemlist(email) {
  return makeRequest(
    `/api/campaigns/${CAMPAIGN_ID}/leads/${encodeURIComponent(email)}`,
    'DELETE'
  );
}

async function cleanup() {
  console.log('\n╔════════════════════════════════════════════════════════════════════╗');
  console.log('║              LEAD CLEANUP - Fix Bad Names & Emails               ║');
  console.log('╚════════════════════════════════════════════════════════════════════╝\n');

  // Step 1: Get all leads from DATABASE (Lemlist GET leads API returns empty)
  console.log('Reading leads from database...\n');

  const Database = require('better-sqlite3');
  const dbPath = './ksd/local-outreach/orchestrator/data/businesses.db';
  const db = new Database(dbPath);

  const leads = db.prepare(`
    SELECT id, name, owner_first_name, owner_last_name, owner_email,
           category, postcode, email_verified
    FROM businesses
    WHERE owner_email IS NOT NULL AND length(owner_email) > 0
    ORDER BY owner_first_name
  `).all();

  console.log(`Found ${leads.length} leads with emails in database\n`);

  // Step 2: Categorize issues
  const badNames = [];
  const badEmails = [

async function compareScraper() {
  console.log('\n' + '='.repeat(80));
  console.log('SCRAPER COMPARISON: HasData vs Outscraper');
  console.log('='.repeat(80) + '\n');

  // Test params
  const location = 'Bramhall';
  const postcode = 'SK7';
  const businessTypes = ['dentists']; // Small test set
  const limit = 5; // Test with just 5 businesses

  console.log('Test Parameters:');
  console.log(`  Location: ${location} (${postcode})`);
  console.log(`  Business Types: ${businessTypes.join(', ')}`);
  console.log(`  Limit: ${limit} businesses`);
  console.log('\n' + '-'.repeat(80) + '\n');

  // Test 1: HasData
  console.log('🔵 TEST 1: HasData Scraper\n');
  let hasdataResults = [];
  let hasdataError = null;
  let hasdataTime = 0;

  try {
    const startTime = Date.now();
    hasdataResults = await hasdataScraper.scrapeGoogleMaps(location, postcode, businessTypes, true);
    hasdataTime = Date.now() - startTime;

    // Limit to 5 for comparison
    hasdataResults = hasdataResults.

function analyzeResults(results, scraperName) {
  const total = results.length;
  const withPhone = results.filter(b => b.phone).length;
  const withWebsite = results.filter(b => b.website).length;
  const withEmail = results.filter(b => b.email).length;
  const withEmailsFromWebsite = results.filter(b => b.emailsFromWebsite && b.emailsFromWebsite.length > 0).length;
  const withRating = results.filter(b => b.rating).length;
  const avgRating = results.filter(b => b.rating).reduce((sum, b) => sum + b.rating, 0) / withRating || 0;
  const avgReviews = results.reduce((sum, b) => sum + (b.reviewCount || 0), 0) / total || 0;

  return {
    scraper: scraperName,
    total,
    withPhone,
    withWebsite,
    withEmail,
    withEmailsFromWebsite,
    withRating,
    avgRating,
    avgReviews,
    phoneRate: Math.round((withPhone / total) * 100),
    websiteRate: Math.round((withWebsite / total) * 100),
    emailRate: Math.round((withEmail / total) * 100),
    emailsFromWebsiteRate: Math.rou

function printStats(hasdata, outscraper) {
  const fields = [
    { label: 'Businesses Found', hasdataVal: hasdata.total, outscraperVal: outscraper.total },
    { label: 'With Phone', hasdataVal: `${hasdata.withPhone} (${hasdata.phoneRate}%)`, outscraperVal: `${outscraper.withPhone} (${outscraper.phoneRate}%)` },
    { label: 'With Website', hasdataVal: `${hasdata.withWebsite} (${hasdata.websiteRate}%)`, outscraperVal: `${outscraper.withWebsite} (${outscraper.websiteRate}%)` },
    { label: 'With Email (GMB)', hasdataVal: `${hasdata.withEmail} (${hasdata.emailRate}%)`, outscraperVal: `${outscraper.withEmail} (${outscraper.emailRate}%)` },
    { label: 'With Website Emails', hasdataVal: `${hasdata.withEmailsFromWebsite} (${hasdata.emailsFromWebsiteRate}%)`, outscraperVal: `${outscraper.withEmailsFromWebsite} (${outscraper.emailsFromWebsiteRate}%)` },
    { label: 'Avg Rating', hasdataVal: hasdata.avgRating.toFixed(1), outscraperVal: outscraper.avgRating.toFixed(1) },
    { label: 'Avg R

async function deleteLead(campaignId, email) {
  const apiKey = getCredential("lemlist", "apiKey");
  const authString = Buffer.from(":" + apiKey).toString("base64");

  return new Promise((resolve, reject) => {
    const options = {
      hostname: 'api.lemlist.com',
      path: `/api/campaigns/${campaignId}/leads/${encodeURIComponent(email)}`,
      method: "DELETE",
      headers: {
        "Authorization": `Basic ${authString}`,
        "Accept": "application/json"
      }
    };

    const req = https.request(options, (res) => {
      const chunks = [];
      let totalLength = 0;

      res.on("data", (chunk) => {
        chunks.push(chunk);
        totalLength += chunk.length;
      });

      res.on("end", () => {
        const data = Buffer.concat(chunks, totalLength).toString('utf8');
        if (res.statusCode >= 200 && res.statusCode < 300) {
          resolve({ success: true, email });
        } else {
          reject(new Error(`Failed to delete ${email}: ${data}`));
     

async function cleanupEmptyLeads() {
  console.log('\n╔════════════════════════════════════════════════════════════════════╗');
  console.log('║              CLEANUP: Delete Empty Leads from Lemlist              ║');
  console.log('╚════════════════════════════════════════════════════════════════════╝\n');

  try {
    console.log('📥 Fetching all leads from campaign...\n');
    const leads = await getLeadsFromCampaign(CAMPAIGN_ID);

    console.log(`Found ${leads.length} total leads\n`);

    // Find empty leads (no firstName or no email)
    const emptyLeads = leads.filter(lead =>
      !lead.firstName ||
      !lead.email ||
      lead.firstName === 'undefined' ||
      lead.email.includes('undefined')
    );

    console.log(`🗑️  Found ${emptyLeads.length} empty leads to delete:\n`);

    if (emptyLeads.length === 0) {
      console.log('✅ No empty leads found! Campaign is clean.\n');
      return;
    }

    for (const lead of emptyLeads) {
      console.log(`   - ${lead.email || '

async function demo() {
  console.log('\n╔════════════════════════════════════════════════════════════════════╗');
  console.log('║           DEMO: EMAIL EXTRACTION FROM WEBSITE                      ║');
  console.log('╚════════════════════════════════════════════════════════════════════╝\n');

  const testWebsite = 'https://www.arundeldentalpractice.co.uk';

  console.log(`🔍 Scraping: ${testWebsite}\n`);
  console.log('Looking for:');
  console.log('  - Email addresses on the website');
  console.log('  - Owner/staff names');
  console.log('  - Matching emails to people\n');

  try {
    const websiteData = await scrapeWebsite(testWebsite);

    console.log('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━');
    console.log('📧 RAW EMAILS FOUND ON WEBSITE');
    console.log('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n');

    if (websiteData.emails && websiteData.emails.length > 0) {
      console.log(`✅ Found ${websiteData.emails.length} ema

function withTimeout(promise, ms, label) {
  return Promise.race([
    promise,
    new Promise((_, reject) =>
      setTimeout(() => reject(new Error(`TIMEOUT after ${ms / 1000}s: ${label}`)), ms)
    ),
  ]);
}

async function main() {
  console.log(`\n=== Enrichment: ${CAMPAIGN} ===\n`);

  initDatabase();

  // Load businesses for this campaign
  const allBusinesses = loadBusinesses({ campaign: CAMPAIGN });
  console.log(`Total in campaign: ${allBusinesses.length}`);

  // Filter to those with websites and not yet enriched
  let toEnrich = allBusinesses.filter(b => {
    const biz = b.business || {};
    const hasWebsite = biz.website &&
      !biz.website.includes('facebook.com') &&
      !biz.website.includes('instagram.com') &&
      !biz.website.includes('twitter.com');
    // Skip already enriched (has email or has been through LLM)
    const alreadyEnriched = biz.ownerFirstName || biz.ownerEmail || biz.email;
    return hasWebsite && !alreadyEnriched;
  });

  if (LIMIT) toEnrich = toEnrich.slice(0, LIMIT);

  console.log(`With website (not yet enriched): ${toEnrich.length}`);
  if (LIMIT) console.log(`Limited to: ${LIMIT}`);

  if (DRY_RUN) {
    console.log('\nDRY RUN — would enrich:

async function searchOutscraper(query, location) {
  const fullQuery = `${query} ${location.name.toLowerCase()}, ${location.postcode.toLowerCase()}`;
  const url = `https://api.outscraper.com/maps/search-v3?query=${encodeURIComponent(fullQuery)}&limit=500`;

  console.log(`  Submitting: "${fullQuery}"`);

  const submitRes = await fetch(url, {
    headers: { 'X-API-KEY': API_KEY }
  });

  if (!submitRes.ok) {
    const text = await submitRes.text();
    throw new Error(`Submit failed (${submitRes.status}): ${text}`);
  }

  const submitData = await submitRes.json();
  const jobId = submitData.id;
  console.log(`  Job ID: ${jobId} — polling...`);

  // Poll for results
  let delay = 2000;
  const maxAttempts = 30;
  for (let attempt = 1; attempt <= maxAttempts; attempt++) {
    await new Promise(r => setTimeout(r, delay));

    const pollRes = await fetch(`https://api.outscraper.cloud/requests/${jobId}`, {
      headers: { 'X-API-KEY': API_KEY }
    });

    if (!pollRes.ok) {
      co

async function main() {
  console.log(`\n=== Football Clubs/Academies — ${CAMPAIGN} (${AREA_ARG}) ===\n`);
  console.log(`Area: ${AREA_ARG} — ${LOCATIONS.length} locations`);

  const totalQueries = SEARCH_TERMS.length * LOCATIONS.length;

  if (DRY_RUN) {
    console.log('DRY RUN — no API calls will be made\n');
    for (const loc of LOCATIONS) {
      console.log(`Location: ${loc.name} (${loc.postcode})`);
      for (const term of SEARCH_TERMS) {
        console.log(`  - "${term} ${loc.name.toLowerCase()}, ${loc.postcode.toLowerCase()}"`);
      }
      console.log('');
    }
    console.log(`Total queries: ${totalQueries}`);
    console.log(`Estimated cost: ~$${(totalQueries * 30 * 0.002).toFixed(2)} (assuming ~30 results each)`);
    console.log(`Campaign tag: ${CAMPAIGN}`);
    console.log('\nRun without --dry-run to execute.\n');
    return;
  }

  // Initialize DB
  initDatabase();

  const allResults = new Map(); // placeId -> business (dedup across locations + terms)
  const p

function isJunkEmail(email) {
  if (!email) return false;
  return JUNK_EMAIL_PATTERNS.some(p => p.test(email));
}

function isJunkContactName(firstName, lastName) {
  const fullName = [firstName, lastName].filter(Boolean).join(' ').trim();
  if (!fullName) return false;
  return JUNK_NAME_PATTERNS.some(p => p.test(fullName));
}

function isNonFootballBusiness(name, category) {
  const lowerName = (name || '').toLowerCase();
  const lowerCat = (category || '').toLowerCase();

  // Check name keywords
  if (NON_FOOTBALL_NAME_KEYWORDS.some(kw => lowerName.includes(kw))) return true;

  // Check category
  if (NON_FOOTBALL_CATEGORIES.some(kw => lowerCat.includes(kw))) return true;

  return false;
}

function cleanBusiness(b) {
  const biz = b.business || {};
  const email = biz.ownerEmail || biz.email || '';
  const name = biz.name || biz.businessName || '';
  const category = biz.category || '';

  // Check for junk email
  if (isJunkEmail(email)) return null;

  // Check for non-football business
  if (isNonFootballBusiness(name, category)) return null;

  // Clean garbage contact names (blank them out rather than reject the business)
  if (isJunkContactName(biz.ownerFirstName, biz.ownerLastName)) {
    biz.ownerFirstName = '';
    biz.ownerLastName = '';
  }

  return b;
}

async function main() {
  initDatabase();

  // List mode — show all campaigns with counts
  if (LIST_MODE) {
    const campaigns = listCampaigns();
    console.log('\n=== Campaigns in Database ===\n');
    if (campaigns.length === 0) {
      console.log('  No campaigns found.');
    } else {
      for (const campaign of campaigns) {
        const stats = getBusinessStats({ campaign });
        console.log(`  ${campaign}`);
        console.log(`    Total: ${stats.total} | Email: ${stats.withEmail} | Exported: ${stats.exported}`);
      }
    }
    console.log('');
    closeDatabase();
    return;
  }

  if (!CAMPAIGN) {
    console.error('ERROR: --campaign=<name> is required. Use --list to see available campaigns.');
    process.exit(1);
  }

  // Load businesses for this campaign
  const filters = { campaign: CAMPAIGN };
  if (HAS_EMAIL) filters.hasEmail = true;
  const allBusinesses = loadBusinesses(filters);

  // Apply additional filters
  let businesses = allBusinesses;
  if (HAS_

function exportCSV(businesses) {
  const headers = [
    'name', 'category', 'address', 'postcode', 'phone', 'website',
    'email', 'email_source', 'email_verified',
    'contact_first_name', 'contact_last_name',
    'rating', 'reviews',
    'revenue_band', 'tier',
    'campaigns'
  ];

  const rows = businesses.map(b => {
    const biz = b.business || {};
    return [
      escapeCsv(biz.name || biz.businessName || ''),
      escapeCsv(biz.category || ''),
      escapeCsv(biz.address || b.postcode || ''),
      escapeCsv(biz.postcode || b.postcode || ''),
      escapeCsv(biz.phone || ''),
      escapeCsv(biz.website || ''),
      escapeCsv(biz.ownerEmail || biz.email || ''),
      escapeCsv(biz.emailSource || ''),
      biz.emailVerified ? 'yes' : 'no',
      escapeCsv(biz.ownerFirstName || ''),
      escapeCsv(biz.ownerLastName || ''),
      biz.rating || '',
      biz.reviewCount || '',
      escapeCsv(biz.revenueBand || ''),
      biz.assignedOfferTier || '',
      escapeCsv(Array

function exportMailead(businesses) {
  // Mailead uses simple CSV: email, first_name, last_name, company_name, custom fields
  const headers = [
    'email', 'first_name', 'last_name', 'company_name', 'phone', 'website', 'category', 'postcode'
  ];

  const rows = businesses.map(b => {
    const biz = b.business || {};
    return [
      escapeCsv(biz.ownerEmail || biz.email || ''),
      escapeCsv(biz.ownerFirstName || ''),
      escapeCsv(biz.ownerLastName || ''),
      escapeCsv(biz.name || biz.businessName || ''),
      escapeCsv(biz.phone || ''),
      escapeCsv(biz.website || ''),
      escapeCsv(biz.category || ''),
      escapeCsv(biz.postcode || b.postcode || ''),
    ];
  });

  const csv = [headers.join(','), ...rows.map(r => r.join(','))].join('\n');

  const timestamp = new Date().toISOString().slice(0, 10);
  const filename = `${CAMPAIGN}-mailead-${timestamp}.csv`;
  const outputPath = path.join(__dirname, 'exports', filename);
  fs.mkdirSync(path.dirname(outputPath), { r

function escapeCsv(val) {
  if (val === null || val === undefined) return '';
  const str = String(val);
  if (str.includes(',') || str.includes('"') || str.includes('\n')) {
    return `"${str.replace(/"/g, '""')}"`;
  }
  return str;
}

async function exportLemlist(businesses) {
  // Dynamic import of lemlist exporter
  const { addLeadToCampaign } = require('./shared/outreach-core/export-managers/lemlist-exporter');

  let exported = 0;
  let skipped = 0;
  let errors = 0;

  for (const b of businesses) {
    const biz = b.business || {};
    const email = biz.ownerEmail || biz.email;
    if (!email) { skipped++; continue; }

    try {
      const lead = {
        email,
        firstName: biz.ownerFirstName || '',
        lastName: biz.ownerLastName || '',
        companyName: biz.name || biz.businessName || '',
        phone: biz.phone || '',
      };

      // Add merge variables if they exist
      if (biz.mergeVariables) {
        Object.assign(lead, biz.mergeVariables);
      }

      await addLeadToCampaign(LEMLIST_CAMPAIGN_ID, lead);
      exported++;
    } catch (err) {
      if (err.message?.includes('DUPLICATE')) {
        skipped++;
      } else {
        errors++;
        console.error(`  Error exporting 

function parseCSV(text) {
  const lines = text.split('\n').filter(l => l.trim());
  if (lines.length < 2) return { headers: [], rows: [] };

  const parseLine = (line) => {
    const fields = [];
    let current = '';
    let inQuotes = false;

    for (let i = 0; i < line.length; i++) {
      const char = line[i];
      if (char === '"') {
        if (inQuotes && line[i + 1] === '"') {
          current += '"';
          i++; // skip escaped quote
        } else {
          inQuotes = !inQuotes;
        }
      } else if (char === ',' && !inQuotes) {
        fields.push(current.trim());
        current = '';
      } else {
        current += char;
      }
    }
    fields.push(current.trim());
    return fields;
  };

  const headers = parseLine(lines[0]).map(h => h.replace(/^"(.*)"$/, '$1').trim());
  const rows = [];

  for (let i = 1; i < lines.length; i++) {
    const values = parseLine(lines[i]);
    if (values.length === 0 || (values.length === 1 && !values[0])) continue;

    c

function mapColumns(headers) {
  const mapping = {};
  const lowerHeaders = headers.map(h => h.toLowerCase().trim());

  for (const [field, variants] of Object.entries(COLUMN_MAP)) {
    for (const variant of variants) {
      const idx = lowerHeaders.indexOf(variant);
      if (idx !== -1) {
        mapping[field] = headers[idx]; // Use original header case
        break;
      }
    }
  }

  return mapping;
}

function isValidEmail(email) {
  if (!email || typeof email !== 'string') return false;
  const cleaned = email.trim().toLowerCase();
  if (!/^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(cleaned)) return false;
  // Reject obvious junk
  if (cleaned.includes('example.com') || cleaned.includes('test.com')) return false;
  if (cleaned.includes('sentry') || cleaned.includes('wixpress')) return false;
  if (/^[a-f0-9]{20,}@/.test(cleaned)) return false; // Tracking hashes
  return true;
}

async function main() {
  console.log(`\n=== PressRanger Import: ${CAMPAIGN} ===\n`);
  console.log(`File: ${FILE}`);
  console.log(`Type: ${CONTACT_TYPE}`);
  console.log(`Verify emails: ${VERIFY ? 'Yes (Reoon)' : 'No'}`);
  console.log(`Mode: ${DRY_RUN ? 'DRY RUN' : 'LIVE'}`);

  // Read CSV
  const filePath = path.resolve(FILE);
  if (!fs.existsSync(filePath)) {
    console.error(`\nERROR: File not found: ${filePath}`);
    process.exit(1);
  }

  const csvText = fs.readFileSync(filePath, 'utf8');
  const { headers, rows } = parseCSV(csvText);

  console.log(`\nCSV loaded: ${rows.length} rows, ${headers.length} columns`);
  console.log(`Headers: ${headers.join(', ')}`);

  // Auto-detect column mapping
  const colMap = mapColumns(headers);
  console.log('\nColumn mapping detected:');
  for (const [field, header] of Object.entries(colMap)) {
    console.log(`  ${field.padEnd(20)} → "${header}"`);
  }

  // Check for unmapped headers (show what we're ignoring)
  const mappedHeaders = 

function getDomain(url) {
  try {
    return new URL(url.startsWith('http') ? url : `https://${url}`).hostname.replace(/^www\./, '');
  } catch {
    return null;
  }
}

function isGenericEmail(email) {
  if (!email) return true;
  const prefix = email.split('@')[0].toLowerCase();
  const genericPrefixes = ['info', 'hello', 'contact', 'enquiries', 'enquiry', 'admin',
    'office', 'sales', 'mail', 'support', 'team', 'help', 'service', 'bookings',
    'booking', 'appointments', 'reception', 'general'];
  return genericPrefixes.some(g => prefix === g || prefix.startsWith(g + '.'));
}

async function runLlmExtraction(db, businesses) {
  console.log('\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━');
  console.log('STAGE 1: LLM EMAIL EXTRACTION (from website text)');
  console.log('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n');

  // Target: businesses with no email OR only generic email, that have a website
  const targets = businesses.filter(b =>
    b.website && (!b.owner_email || isGenericEmail(b.owner_email)) &&
    !b.website.includes('facebook.com') && !b.website.includes('instagram.com')
  );

  console.log(`  Targets: ${targets.length} businesses (no email or generic email)\n`);
  if (DRY_RUN) return;

  for (const biz of targets) {
    const label = biz.name.substring(0, 40).padEnd(40);
    try {
      const text = await fetchWebsiteText(biz.website);
      if (!text) {
        console.log(`  NOFETCH  ${label}`);
        continue;
      }

      const result = await llmExtractOwners(biz.name, text);
      STATS.llm.processed++;

async function runPatternGuessing(db, businesses) {
  console.log('\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━');
  console.log('STAGE 2: PATTERN GUESSING + REOON VERIFICATION');
  console.log('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n');

  // Re-read businesses to get updated state after LLM stage
  const fresh = db.prepare(`
    SELECT * FROM businesses
    WHERE owner_first_name IS NOT NULL AND length(owner_first_name) > 0
      AND owner_first_name != 'there'
      AND website IS NOT NULL AND length(website) > 0
      AND (owner_email IS NULL OR length(owner_email) = 0)
    ORDER BY name
  `).all();

  const reoonRemaining = getReoonQuota();
  console.log(`  Targets: ${fresh.length} businesses (have name + website, no email)`);
  console.log(`  Reoon credits remaining: ${reoonRemaining}`);

  // Each business needs up to 7 pattern checks
  const maxBusinesses = Math.floor(reoonRemaining / 3); // Budget ~3 patterns per business (stop on first val

async function runIcypeasFinder(db, businesses) {
  console.log('\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━');
  console.log('STAGE 3: ICYPEAS EMAIL FINDER');
  console.log('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n');

  // Re-read: businesses with name + website but still no email after stages 1-2
  const fresh = db.prepare(`
    SELECT * FROM businesses
    WHERE owner_first_name IS NOT NULL AND length(owner_first_name) > 0
      AND owner_first_name != 'there'
      AND owner_last_name IS NOT NULL AND length(owner_last_name) > 0
      AND website IS NOT NULL AND length(website) > 0
      AND (owner_email IS NULL OR length(owner_email) = 0)
    ORDER BY name
  `).all();

  console.log(`  Targets: ${fresh.length} businesses (have full name + website, still no email)`);
  console.log(`  Icypeas limit: 100/day\n`);

  const toProcess = fresh.slice(0, 100); // Respect daily limit

  if (DRY_RUN) {
    for (const biz of toProcess.slice(0, 10)) {
     

async function main() {
  console.log('\n╔════════════════════════════════════════════════════════════════════╗');
  console.log('║              EMAIL IMPROVEMENT PIPELINE                          ║');
  console.log('╚════════════════════════════════════════════════════════════════════╝');

  const db = new Database(DB_PATH);

  // Get initial state
  let query = `SELECT * FROM businesses WHERE website IS NOT NULL AND length(website) > 0 ORDER BY name`;
  if (LIMIT) query = query.replace('ORDER BY', `ORDER BY`) + ` LIMIT ${LIMIT}`;
  const businesses = db.prepare(query).all();

  const withEmail = businesses.filter(b => b.owner_email && b.owner_email.length > 0);
  const generic = withEmail.filter(b => isGenericEmail(b.owner_email));
  const personal = withEmail.filter(b => !isGenericEmail(b.owner_email));
  const noEmail = businesses.filter(b => !b.owner_email || b.owner_email.length === 0);

  console.log(`\n  BEFORE: ${businesses.length} businesses with websites`);
  console.log(`  

async function processBusinesses(location, postcode, businessTypes = [], extractEmails = true) {
  const scrapedAt = new Date().toISOString();
  
  // Step 1: Scrape Google Maps (with postcode for accuracy)
  // Try Outscraper first, fallback to HasData if it fails OR returns 0 results
  logger.info('main', `Scraping businesses in ${location}${postcode ? ` (${postcode})` : ""}...`);

  let businesses = [];
  let scraperUsed = null;

  try {
    logger.info('main', 'Trying Outscraper API...');
    businesses = await scrapeGoogleMapsOutscraper(location, postcode, businessTypes, extractEmails);
    scraperUsed = 'outscraper';
    logger.info('main', `Found ${businesses.length} businesses via Outscraper`);

    // If Outscraper returned 0 results, try HasData as backup
    if (businesses.length === 0) {
      logger.info('main', 'Outscraper returned 0 results, trying HasData as backup...');
      try {
        const hasdataResults = await scrapeGoogleMaps(location, postcode, businessTypes,

async function generateAndExport(enrichedBusinesses, config = {}) {
  const approvedTemplates = loadApprovedTemplates();
  const exported = [];
  
  for (const business of enrichedBusinesses) {
    try {
      // Generate content
      const content = await generateOutreachContent(business, {
        provider: process.env.CONTENT_PROVIDER || 'claude', // Use Claude by default
        generateEmail: true,
        generateLinkedIn: !!business.linkedInUrl,
        emailSequence: true
      });
      
      // Check if approval needed
      if (needsApproval(business, approvedTemplates)) {
        addToApprovalQueue(business, content.email || content.emailSequence[0]);
        logger.info('main', `Added ${business.category} email to approval queue`);
        continue; // Skip export until approved
      }
      
      const exportedTo = [];
      
      // Export to Lemlist
      if (business.ownerEmail && config.lemlistCampaignId) {
        await exportToLemlist(business, config.lemlistCa

function isValidLocation(location) {
  if (!location || typeof location !== "string") return false;
  // Allow alphanumeric, spaces, hyphens, apostrophes (for places like "Bishop's Stortford")
  const locationPattern = /^[a-zA-Z0-9\s\-']+$/;
  return locationPattern.test(location) && location.length >= 2 && location.length <= 100;
}

function isValidPostcode(postcode) {
  if (!postcode || typeof postcode !== "string") return false;
  // UK postcode prefix pattern (e.g., SK7, M1, SW1A, EC1A)
  const postcodePattern = /^[A-Z]{1,2}[0-9][0-9A-Z]?$/i;
  return postcodePattern.test(postcode.trim());
}

function sanitizeBusinessTypes(input) {
  if (!input || typeof input !== "string") return [];
  return input
    .split(",")
    .map(type => type.trim().toLowerCase())
    .filter(type => type.length > 0 && type.length <= 50 && /^[a-zA-Z0-9\s\-]+$/.test(type));
}

function printUsage() {
  console.log(`
Usage: node main.js [location] [postcode] [businessTypes] [options]

Arguments:
  location       Location name (e.g., "Bramhall", "Manchester")
  postcode       UK postcode prefix (e.g., "SK7", "M1")
  businessTypes  Comma-separated list (e.g., "restaurants,cafes")

Options:
  --no-emails    Skip email extraction from websites
  --load         Load existing businesses instead of scraping
  --help         Show this help message

Examples:
  node main.js Bramhall SK7
  node main.js "Manchester" M1 "restaurants,bars" --no-emails
  node main.js --load Bramhall SK7
`);
}

function loadAgreements() {
  try {
    if (fs.existsSync(AGREEMENTS_FILE)) {
      const data = fs.readFileSync(AGREEMENTS_FILE, "utf8");
      return JSON.parse(data);
    }
    return {};
  } catch (error) {
    logger.warn('barter-agreements', 'Failed to load barter agreements', { error: error.message });
    return {};
  }
}

function saveAgreements(agreements) {
  const dir = path.dirname(AGREEMENTS_FILE);
  if (!fs.existsSync(dir)) {
    fs.mkdirSync(dir, { recursive: true });
  }
  fs.writeFileSync(AGREEMENTS_FILE, JSON.stringify(agreements, null, 2));
}

function hasAgreement(category) {
  const agreements = loadAgreements();
  const categoryKey = (category || "").toLowerCase();
  return agreements[categoryKey] && agreements[categoryKey].length > 0;
}