Function bodies 406 total

function getAgreements(category) {
  const agreements = loadAgreements();
  const categoryKey = (category || "").toLowerCase();
  return agreements[categoryKey] || [];
}

function addAgreement(category, businessName) {
  const agreements = loadAgreements();
  const categoryKey = (category || "").toLowerCase();
  
  if (!agreements[categoryKey]) {
    agreements[categoryKey] = [];
  }
  
  // Avoid duplicates
  if (!agreements[categoryKey].includes(businessName)) {
    agreements[categoryKey].push(businessName);
    saveAgreements(agreements);
    return true;
  }
  
  return false;
}

function removeAgreement(category, businessName) {
  const agreements = loadAgreements();
  const categoryKey = (category || "").toLowerCase();
  
  if (agreements[categoryKey]) {
    agreements[categoryKey] = agreements[categoryKey].filter(
      name => name !== businessName
    );
    
    // Remove category if empty
    if (agreements[categoryKey].length === 0) {
      delete agreements[categoryKey];
    }
    
    saveAgreements(agreements);
    return true;
  }
  
  return false;
}

function getAllAgreements() {
  return loadAgreements();
}

function detectBarterOpportunity(business) {
  const category = (business.category || "").toLowerCase();
  const barterInfo = Object.entries(barterCategories).find(([cat]) => category.includes(cat));
  
  if (!barterInfo) {
    return { eligible: false, available: false, value: "low" };
  }
  
  const [barterCategory, info] = barterInfo;
  
  // Check if this category allows multiple agreements
  const allowsMultiple = MULTIPLE_ALLOWED.some(cat => barterCategory.includes(cat));
  const isSingleOnly = SINGLE_ONLY.some(cat => barterCategory.includes(cat));
  
  // For single-only categories, check if agreement exists
  // For multiple-allowed categories, always available
  let hasExistingAgreement = false;
  if (isSingleOnly) {
    hasExistingAgreement = hasAgreement(barterCategory);
  }
  // For multiple-allowed, we don't check - can have multiple
  
  return {
    eligible: true,
    available: !hasExistingAgreement,
    category: barterCategory,
    value: info.value,
    offering: in

function loadChainBrands() {
  // Return cached data if valid
  if (cachedChains && cacheTimestamp && (Date.now() - cacheTimestamp < CACHE_TTL_MS)) {
    return cachedChains;
  }

  try {
    const data = fs.readFileSync(CHAINS_CONFIG_PATH, "utf8");
    const config = JSON.parse(data);
    
    // Update cache
    cachedChains = config.chains || [];
    cachedSignals = config.signals || {};
    cacheTimestamp = Date.now();
    
    return cachedChains;
  } catch (error) {
    logger.warn('chain-filter', 'Failed to load chain brands config', { error: error.message });
    return [];
  }
}

function loadChainSignals() {
  // Return cached data if valid
  if (cachedSignals && cacheTimestamp && (Date.now() - cacheTimestamp < CACHE_TTL_MS)) {
    return cachedSignals;
  }

  try {
    const data = fs.readFileSync(CHAINS_CONFIG_PATH, "utf8");
    const config = JSON.parse(data);
    
    // Update cache
    cachedChains = config.chains || [];
    cachedSignals = config.signals || {};
    cacheTimestamp = Date.now();
    
    return cachedSignals;
  } catch (error) {
    logger.warn('chain-filter', 'Failed to load chain signals config', { error: error.message });
    return {};
  }
}

function clearCache() {
  cachedChains = null;
  cachedSignals = null;
  cacheTimestamp = null;
  logger.debug('chain-filter', 'Cache cleared');
}

function isChain(business) {
  const businessName = (business.name || business.businessName || "").toLowerCase();
  const chains = loadChainBrands();
  
  // Check if name contains any chain brand
  const isChainBrand = chains.some(chain => businessName.includes(chain.toLowerCase()));
  
  // Check Google Maps signals
  const reviewCount = business.reviewCount || 0;
  const locationCount = business.locationCount || 1;
  const signals = loadChainSignals();
  
  const hasChainSignals = 
    reviewCount >= (signals.minReviewCount || 1000) ||
    locationCount >= (signals.minLocations || 2);
  
  return isChainBrand || hasChainSignals;
}

function isPub(business) {
  const businessName = (business.name || business.businessName || "").toLowerCase();
  const website = (business.website || "").toLowerCase();
  const address = (business.address || "").toLowerCase();
  
  // Check for pub keywords in name
  const pubKeywords = ["pub", "tavern", "inn", "bar", "alehouse", "public house"];
  const hasPubKeyword = pubKeywords.some(keyword => 
    businessName.includes(keyword) || address.includes(keyword)
  );
  
  // Check for pub-related domains
  const pubDomains = [
    "greeneking.co.uk",
    "robinsonsbrewery.com",
    "pubs",
    "brewery",
    "alehouse"
  ];
  const hasPubDomain = pubDomains.some(domain => website.includes(domain));
  
  return hasPubKeyword || hasPubDomain;
}

function filterChains(businesses) {
  return businesses.filter(business => !isChain(business) && !isPub(business));
}

async function searchCompany(companyName, postcode) {
  const apiKey = getCredential("companiesHouse", "apiKey");
  
  return new Promise((resolve, reject) => {
    const searchQuery = encodeURIComponent(companyName);
    const path = postcode 
      ? `/search/companies?q=${searchQuery}&items_per_page=10`
      : `/search/companies?q=${searchQuery}&items_per_page=10`;
    
    const options = {
      hostname: COMPANIES_HOUSE_BASE_URL,
      path: path,
      method: "GET",
      headers: {
        "Authorization": Buffer.from(apiKey + ":").toString("base64"),
        "Accept": "application/json"
      }
    };
    
    const req = https.request(options, (res) => {
      let data = "";
      
      res.on("data", (chunk) => {
        data += chunk;
      });
      
      res.on("end", () => {
        try {
          const result = JSON.parse(data);
          
          if (result.items && result.items.length > 0) {
            resolve(result.items);
          } else {
            reso

async function getCompanyOfficers(companyNumber) {
  const apiKey = getCredential("companiesHouse", "apiKey");
  
  return new Promise((resolve, reject) => {
    const options = {
      hostname: COMPANIES_HOUSE_BASE_URL,
      path: `/company/${companyNumber}/officers`,
      method: "GET",
      headers: {
        "Authorization": Buffer.from(apiKey + ":").toString("base64"),
        "Accept": "application/json"
      }
    };
    
    const req = https.request(options, (res) => {
      let data = "";
      
      res.on("data", (chunk) => {
        data += chunk;
      });
      
      res.on("end", () => {
        try {
          const result = JSON.parse(data);
          
          if (result.items && result.items.length > 0) {
            // Filter for active directors/officers
            const activeOfficers = result.items.filter(o => 
              o.officer_role === "director" || 
              o.officer_role === "secretary" ||
              o.resigned_on === undefined
      

async function getOwnerByRegistrationNumber(registrationNumber) {
  try {
    if (!registrationNumber || !/^\d{8}$/.test(registrationNumber)) {
      logger.warn('companies-house', 'Invalid registration number format', { registrationNumber });
      return null;
    }

    logger.info('companies-house', 'Looking up company by registration number', { registrationNumber });

    // Get officers directly using registration number
    const officers = await getCompanyOfficers(registrationNumber);

    if (officers.length === 0) {
      logger.info('companies-house', 'No officers found for registration number', { registrationNumber });
      return null;
    }

    // Return first active director/officer
    const owner = officers[0];
    const { firstName, lastName } = parseName(owner.name);

    // Skip if name validation failed (e.g., job title instead of person name)
    if (!firstName) {
      logger.warn('companies-house', 'Name validation failed for officer', {
        registrationNu

async function getAllOwnersByRegistrationNumber(registrationNumber, maxOwners = 5) {
  try {
    if (!registrationNumber || !/^\d{8}$/.test(registrationNumber)) {
      logger.warn('companies-house', 'Invalid registration number format', { registrationNumber });
      return [];
    }

    logger.info('companies-house', 'Looking up all owners by registration number', { registrationNumber, maxOwners });

    // Get officers directly using registration number
    const officers = await getCompanyOfficers(registrationNumber);

    if (officers.length === 0) {
      logger.info('companies-house', 'No officers found for registration number', { registrationNumber });
      return [];
    }

    // Convert all officers to owner format (up to maxOwners)
    const owners = officers.slice(0, maxOwners).map(officer => {
      const { firstName, lastName } = parseName(officer.name);
      return {
        firstName: firstName,
        lastName: lastName,
        fullName: officer.name,
        tit

async function getOwnerName(businessName, postcode) {
  try {
    // Search for company
    const companies = await searchCompany(businessName, postcode);

    if (companies.length === 0) {
      return null;
    }

    // Try first match (most likely)
    const company = companies[0];

    // Get officers
    const officers = await getCompanyOfficers(company.company_number);

    if (officers.length === 0) {
      return null;
    }

    // Return first active director/officer
    const owner = officers[0];
    const { firstName, lastName } = parseName(owner.name);

    // Skip if name validation failed (e.g., job title instead of person name)
    if (!firstName) {
      logger.warn('companies-house', 'Name validation failed for officer', {
        businessName,
        rawName: owner.name,
        reason: 'parseName returned empty (likely job title or invalid name)'
      });
      return null;
    }

    return {
      firstName: firstName,
      lastName: lastName,
      fullName: 

async function getAllOwnersByName(businessName, postcode, maxOwners = 5) {
  try {
    // Search for company
    const companies = await searchCompany(businessName, postcode);

    if (companies.length === 0) {
      return [];
    }

    // Try first match (most likely)
    const company = companies[0];

    // Get officers
    const officers = await getCompanyOfficers(company.company_number);

    if (officers.length === 0) {
      return [];
    }

    // Convert all officers to owner format (up to maxOwners)
    const owners = officers.slice(0, maxOwners).map(officer => {
      const { firstName, lastName } = parseName(officer.name);
      return {
        firstName: firstName,
        lastName: lastName,
        fullName: officer.name,
        title: officer.officer_role,
        companyNumber: company.company_number,
        companyName: company.title,
        source: 'companies-house-name-search'
      };
    }).filter(owner => owner.firstName); // Remove invalid names that fail

function isValidSQLiteFile(filePath) {
  try {
    const stat = fs.statSync(filePath);
    if (stat.size < 100) return false; // SQLite header is 100 bytes
    const fd = fs.openSync(filePath, 'r');
    const buf = Buffer.alloc(16);
    fs.readSync(fd, buf, 0, 16, 0);
    fs.closeSync(fd);
    return buf.toString('ascii', 0, 15) === 'SQLite format 3';
  } catch (e) {
    return false;
  }
}

function backupDatabase() {
  if (!fs.existsSync(DB_PATH)) return null;
  if (!isValidSQLiteFile(DB_PATH)) return null;

  const stat = fs.statSync(DB_PATH);
  if (stat.size === 0) return null;

  const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
  const backupName = `businesses-${timestamp}.db`;
  const backupPath = path.join(BACKUP_DIR, backupName);

  try {
    fs.copyFileSync(DB_PATH, backupPath);

    // Also backup WAL if it exists (contains uncommitted data)
    const walPath = DB_PATH + '-wal';
    if (fs.existsSync(walPath) && fs.statSync(walPath).size > 0) {
      fs.copyFileSync(walPath, backupPath + '-wal');
    }

    rotateBackups();
    return backupPath;
  } catch (e) {
    console.error(`[DB] Backup failed: ${e.message}`);
    return null;
  }
}

function rotateBackups() {
  try {
    const files = fs.readdirSync(BACKUP_DIR)
      .filter(f => f.startsWith('businesses-') && f.endsWith('.db'))
      .sort()
      .reverse();

    // Remove old backups beyond MAX_BACKUPS
    for (let i = MAX_BACKUPS; i < files.length; i++) {
      const toDelete = path.join(BACKUP_DIR, files[i]);
      fs.unlinkSync(toDelete);
      // Also clean up associated WAL file
      const walFile = toDelete + '-wal';
      if (fs.existsSync(walFile)) fs.unlinkSync(walFile);
    }
  } catch (e) {
    // Non-fatal — don't block database operations for cleanup failures
  }
}

function restoreLatestBackup() {
  try {
    const files = fs.readdirSync(BACKUP_DIR)
      .filter(f => f.startsWith('businesses-') && f.endsWith('.db') && !f.endsWith('-wal'))
      .sort()
      .reverse();

    for (const file of files) {
      const backupPath = path.join(BACKUP_DIR, file);
      if (isValidSQLiteFile(backupPath)) {
        fs.copyFileSync(backupPath, DB_PATH);
        // Restore WAL if it exists
        const walBackup = backupPath + '-wal';
        const walPath = DB_PATH + '-wal';
        if (fs.existsSync(walBackup)) {
          fs.copyFileSync(walBackup, walPath);
        } else if (fs.existsSync(walPath)) {
          fs.unlinkSync(walPath); // Remove stale WAL
        }
        console.log(`[DB] Restored from backup: ${file}`);
        return backupPath;
      }
    }
  } catch (e) {
    console.error(`[DB] Restore failed: ${e.message}`);
  }
  return null;
}

function initDatabase() {
  if (db) return db;

  // If DB file exists but is 0 bytes or corrupt, try to restore from backup
  if (fs.existsSync(DB_PATH)) {
    const stat = fs.statSync(DB_PATH);
    if (stat.size === 0 || !isValidSQLiteFile(DB_PATH)) {
      console.warn(`[DB] WARNING: Database file is ${stat.size === 0 ? 'empty (0 bytes)' : 'corrupt'}!`);
      const restored = restoreLatestBackup();
      if (restored) {
        console.log('[DB] Successfully restored from backup.');
      } else {
        console.warn('[DB] No valid backup found. Creating fresh database.');
      }
    }
  }

  // Backup existing database before opening (if it has data)
  const backupPath = backupDatabase();
  if (backupPath) {
    console.log(`[DB] Backup created: ${path.basename(backupPath)}`);
  }

  db = new Database(DB_PATH);
  db.pragma("journal_mode = WAL");

  // Execute schema statements
  for (const statement of SCHEMA_STATEMENTS) {
    db.exec(statement);
  }

  // Migration: add campaig

function generateBusinessId(business) {
  const name = (business.name || business.businessName || "unknown").toLowerCase().replace(/[^a-z0-9]+/g, "-").substring(0, 50);
  const postcode = (business.postcode || "").toLowerCase().replace(/\s+/g, "");
  const hash = crypto.createHash("md5").update(JSON.stringify({ name: business.name, address: business.address })).digest("hex").substring(0, 8);
  return `${name}-${postcode}-${hash}`;
}

function checkDuplicate(business) {
  const database = initDatabase();
  const byName = database.prepare(`SELECT id FROM businesses WHERE name = ? AND postcode = ? ORDER BY created_at DESC LIMIT 1`).get(business.name || business.businessName || "Unknown Business", business.postcode || "");
  if (byName) return byName.id;
  if (business.website) {
    try {
      const domain = new URL(business.website).hostname.replace("www.", "");
      const byWebsite = database.prepare(`SELECT id FROM businesses WHERE website LIKE ? ORDER BY created_at DESC LIMIT 1`).get(`%${domain}%`);
      if (byWebsite) return byWebsite.id;
    } catch (e) {}
  }
  if (business.address) {
    const byAddress = database.prepare(`SELECT id FROM businesses WHERE address = ? ORDER BY created_at DESC LIMIT 1`).get(business.address);
    if (byAddress) return byAddress.id;
  }
  return null;
}

function saveBusiness(business, metadata = {}) {
  const database = initDatabase();
  const businessId = generateBusinessId(business);
  const existingId = checkDuplicate(business);
  const finalId = existingId || businessId;
  const stmt = database.prepare(`INSERT INTO businesses (id, name, location, postcode, address, website, phone, category, rating, review_count, owner_first_name, owner_last_name, owner_email, email_source, email_verified, linkedin_url, estimated_revenue, revenue_band, revenue_confidence, assigned_tier, setup_fee, monthly_price, ghl_offer, lead_magnet, barter_opportunity, status, scraped_at, enriched_at, exported_to, exported_at, business_data) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(id) DO UPDATE SET name = excluded.name, location = excluded.location, postcode = excluded.postcode, address = excluded.address, website = excluded.website, phone = excluded.phone, category = excluded.category, rat

function batchSaveBusinesses(businesses, metadata = {}) {
  const database = initDatabase();
  const transaction = database.transaction((businesses, metadata) => { for (const business of businesses) { saveBusiness(business, metadata); } });
  transaction(businesses, metadata);
  return businesses.length;
}

function loadBusinesses(filters = {}) {
  const database = initDatabase();
  let query = "SELECT * FROM businesses WHERE 1=1";
  const params = [];
  if (filters.location) { query += " AND location = ?"; params.push(filters.location); }
  if (filters.postcode) { query += " AND postcode = ?"; params.push(filters.postcode); }
  if (filters.status) { query += " AND status = ?"; params.push(filters.status); }
  if (filters.tier) { query += " AND assigned_tier = ?"; params.push(filters.tier); }
  if (filters.hasEmail !== undefined) { query += filters.hasEmail ? " AND owner_email IS NOT NULL" : " AND owner_email IS NULL"; }
  if (filters.hasLinkedIn !== undefined) { query += filters.hasLinkedIn ? " AND linkedin_url IS NOT NULL" : " AND linkedin_url IS NULL"; }
  if (filters.enrichedAfter) { query += " AND enriched_at >= ?"; params.push(filters.enrichedAfter); }
  if (filters.enrichedBefore) { query += " AND enriched_at <= ?"; params.push(filters.enrichedBefore); }
  if (filters.campaign) { q

function updateBusiness(businessId, updates) {
  const database = initDatabase();
  const setClauses = [];
  const params = [];
  if (updates.status) { setClauses.push("status = ?"); params.push(updates.status); }
  if (updates.exportedTo) { setClauses.push("exported_to = ?"); params.push(JSON.stringify(updates.exportedTo)); }
  if (updates.exportedAt) { setClauses.push("exported_at = ?"); params.push(updates.exportedAt); }
  if (updates.business) {
    const b = updates.business;
    if (b.ownerEmail !== undefined) { setClauses.push("owner_email = ?"); params.push(b.ownerEmail); }
    if (b.linkedInUrl !== undefined) { setClauses.push("linkedin_url = ?"); params.push(b.linkedInUrl); }
    if (b.assignedOfferTier !== undefined) { setClauses.push("assigned_tier = ?"); params.push(b.assignedOfferTier); }
    const current = database.prepare("SELECT business_data FROM businesses WHERE id = ?").get(businessId);
    if (current && current.business_data) {
      const currentBusiness = JSON.

function getBusiness(businessId) {
  const database = initDatabase();
  const row = database.prepare("SELECT * FROM businesses WHERE id = ?").get(businessId);
  if (!row) return null;
  const business = row.business_data ? JSON.parse(row.business_data) : { name: row.name, postcode: row.postcode, address: row.address, website: row.website, phone: row.phone, category: row.category, rating: row.rating, reviewCount: row.review_count, ownerFirstName: row.owner_first_name, ownerLastName: row.owner_last_name, ownerEmail: row.owner_email, emailSource: row.email_source, emailVerified: !!row.email_verified, linkedInUrl: row.linkedin_url, estimatedRevenue: row.estimated_revenue, revenueBand: row.revenue_band, assignedOfferTier: row.assigned_tier };
  return { id: row.id, scrapedAt: row.scraped_at, enrichedAt: row.enriched_at, location: row.location, postcode: row.postcode, business: business, exportedTo: (() => { try { return row.exported_to ? JSON.parse(row.exported_to) : []; } catch(e) { return

function addCampaignToBusiness(businessId, campaignName) {
  const database = initDatabase();
  const row = database.prepare("SELECT campaigns FROM businesses WHERE id = ?").get(businessId);
  if (!row) return null;
  let current = [];
  try { current = row.campaigns ? JSON.parse(row.campaigns) : []; } catch(e) {}
  if (current.includes(campaignName)) return current;
  current.push(campaignName);
  database.prepare("UPDATE businesses SET campaigns = ?, updated_at = CURRENT_TIMESTAMP WHERE id = ?").run(JSON.stringify(current), businessId);
  return current;
}

function listCampaigns() {
  const database = initDatabase();
  const rows = database.prepare("SELECT DISTINCT campaigns FROM businesses WHERE campaigns IS NOT NULL").all();
  const campaigns = new Set();
  for (const row of rows) {
    try {
      const arr = JSON.parse(row.campaigns);
      arr.forEach(c => campaigns.add(c));
    } catch(e) {}
  }
  return Array.from(campaigns).sort();
}

function getBusinessStats(filters = {}) {
  const database = initDatabase();
  let whereClause = "WHERE 1=1";
  const params = [];
  if (filters.location) { whereClause += " AND location = ?"; params.push(filters.location); }
  if (filters.postcode) { whereClause += " AND postcode = ?"; params.push(filters.postcode); }
  if (filters.status) { whereClause += " AND status = ?"; params.push(filters.status); }
  if (filters.campaign) { whereClause += " AND campaigns LIKE ?"; params.push(`%"${filters.campaign}"%`); }
  const stats = { total: database.prepare(`SELECT COUNT(*) as count FROM businesses ${whereClause}`).get(...params).count, byStatus: {}, byTier: {}, byLocation: {}, withEmail: 0, withLinkedIn: 0, withBoth: 0, exported: 0, dateRange: { earliest: null, latest: null } };
  const statusRows = database.prepare(`SELECT status, COUNT(*) as count FROM businesses ${whereClause} GROUP BY status`).all(...params);
  statusRows.forEach(row => { stats.byStatus[row.status] = row.count; });
  

function batchUpdateBusinesses(updates) {
  const database = initDatabase();
  const transaction = database.transaction((updates) => { for (const update of updates) { updateBusiness(update.id, update.updates); } });
  transaction(updates);
  return updates.length;
}

function closeDatabase() {
  if (db) {
    try {
      // Checkpoint WAL to ensure all data is written to the main database file
      db.pragma("wal_checkpoint(TRUNCATE)");
    } catch (e) {
      // Non-fatal — database may already be in a non-WAL mode
    }
    db.close();
    db = null;
  }
}

function listBackups() {
  try {
    return fs.readdirSync(BACKUP_DIR)
      .filter(f => f.startsWith('businesses-') && f.endsWith('.db') && !f.endsWith('-wal'))
      .sort()
      .reverse()
      .map(f => {
        const stat = fs.statSync(path.join(BACKUP_DIR, f));
        return { file: f, size: stat.size, date: stat.mtime.toISOString() };
      });
  } catch (e) {
    return [];
  }
}

function generateBusinessId(business) {
  const name = (business.name || business.businessName || "unknown").toLowerCase()
    .replace(/[^a-z0-9]+/g, "-")
    .substring(0, 50);
  const postcode = (business.postcode || "").toLowerCase().replace(/\s+/g, "");
  const hash = crypto.createHash("md5")
    .update(JSON.stringify({ name: business.name, address: business.address }))
    .digest("hex")
    .substring(0, 8);
  
  return `${name}-${postcode}-${hash}`;
}

function loadIndex() {
  if (!fs.existsSync(INDEX_FILE)) {
    return {};
  }
  try {
    return JSON.parse(fs.readFileSync(INDEX_FILE, "utf8"));
  } catch (error) {
    return {};
  }
}

function saveIndex(index) {
  fs.writeFileSync(INDEX_FILE, JSON.stringify(index, null, 2));
}

function saveBusiness(business, metadata = {}) {
  const businessId = generateBusinessId(business);
  const filePath = path.join(DATA_DIR, `${businessId}.json`);
  
  const record = {
    id: businessId,
    scrapedAt: metadata.scrapedAt || new Date().toISOString(),
    enrichedAt: metadata.enrichedAt || new Date().toISOString(),
    location: metadata.location || business.location,
    postcode: business.postcode || metadata.postcode,
    business: business,
    exportedTo: metadata.exportedTo || [],
    exportedAt: metadata.exportedAt || null,
    status: metadata.status || "enriched"
  };
  
  // Save business file
  fs.writeFileSync(filePath, JSON.stringify(record, null, 2));
  
  // Update index
  const index = loadIndex();
  index[businessId] = {
    id: businessId,
    name: business.name || business.businessName,
    location: record.location,
    postcode: record.postcode,
    status: record.status,
    enrichedAt: record.enrichedAt,
    exportedAt: record.exportedAt,
    hasE

function loadBusinesses(filters = {}) {
  const index = loadIndex();
  const businesses = [];
  
  for (const [businessId, indexData] of Object.entries(index)) {
    // Apply filters
    if (filters.location && indexData.location !== filters.location) continue;
    if (filters.postcode && indexData.postcode !== filters.postcode) continue;
    if (filters.status && indexData.status !== filters.status) continue;
    if (filters.tier && indexData.tier !== filters.tier) continue;
    if (filters.hasEmail !== undefined && indexData.hasEmail !== filters.hasEmail) continue;
    if (filters.hasLinkedIn !== undefined && indexData.hasLinkedIn !== filters.hasLinkedIn) continue;
    
    // Date filters
    if (filters.enrichedAfter && indexData.enrichedAt < filters.enrichedAfter) continue;
    if (filters.enrichedBefore && indexData.enrichedAt > filters.enrichedBefore) continue;
    
    // Load full business data
    const filePath = path.join(DATA_DIR, `${businessId}.json`);
    if (fs.existsSy

function updateBusiness(businessId, updates) {
  const filePath = path.join(DATA_DIR, `${businessId}.json`);
  
  if (!fs.existsSync(filePath)) {
    throw new Error(`Business ${businessId} not found`);
  }
  
  const record = JSON.parse(fs.readFileSync(filePath, "utf8"));
  
  // Update fields
  Object.assign(record, updates);
  if (updates.business) {
    record.business = { ...record.business, ...updates.business };
  }
  
  // Update timestamps
  if (updates.status) {
    record.status = updates.status;
    if (updates.status === "exported" && !record.exportedAt) {
      record.exportedAt = new Date().toISOString();
    }
  }
  
  // Save updated record
  fs.writeFileSync(filePath, JSON.stringify(record, null, 2));
  
  // Update index
  const index = loadIndex();
  if (index[businessId]) {
    if (updates.status) index[businessId].status = updates.status;
    if (updates.exportedTo) index[businessId].exportedTo = updates.exportedTo;
    if (record.exportedAt) index[businessId].exp

function archiveBusiness(businessId) {
  const filePath = path.join(DATA_DIR, `${businessId}.json`);
  const archivePath = path.join(ARCHIVE_DIR, `${businessId}.json`);
  
  if (!fs.existsSync(filePath)) {
    throw new Error(`Business ${businessId} not found`);
  }
  
  // Move file
  fs.renameSync(filePath, archivePath);
  
  // Remove from index
  const index = loadIndex();
  delete index[businessId];
  saveIndex(index);
  
  return true;
}

function getBusinessStats(filters = {}) {
  const businesses = loadBusinesses(filters);
  const index = loadIndex();
  
  const stats = {
    total: businesses.length,
    byStatus: {},
    byTier: {},
    byLocation: {},
    withEmail: 0,
    withLinkedIn: 0,
    withBoth: 0,
    exported: 0,
    dateRange: {
      earliest: null,
      latest: null
    }
  };
  
  let earliestDate = null;
  let latestDate = null;
  
  businesses.forEach(record => {
    // Status breakdown
    stats.byStatus[record.status] = (stats.byStatus[record.status] || 0) + 1;
    
    // Tier breakdown
    const tier = record.business.assignedOfferTier || "unknown";
    stats.byTier[tier] = (stats.byTier[tier] || 0) + 1;
    
    // Location breakdown
    const location = record.location || "unknown";
    stats.byLocation[location] = (stats.byLocation[location] || 0) + 1;
    
    // Email/LinkedIn stats
    if (record.business.ownerEmail) stats.withEmail++;
    if (record.business.linkedInUrl) stats.withLinked

function getBusiness(businessId) {
  const filePath = path.join(DATA_DIR, `${businessId}.json`);
  
  if (!fs.existsSync(filePath)) {
    return null;
  }
  
  return JSON.parse(fs.readFileSync(filePath, "utf8"));
}

function getBackoffDelay(attempt) {
  const delay = INITIAL_POLL_DELAY_MS * Math.pow(2, Math.min(attempt - 1, 3));
  return Math.min(delay, MAX_POLL_DELAY_MS);
}

async function scrapeGoogleMaps(location, postcode, businessTypes = [], extractEmails = true) {
  const apiKey = getCredential("hasdata", "apiKey");
  
  // Format location with postcode for accuracy: "Location, Postcode" or "CUSTOM>Location, Postcode"
  // This ensures we get the right location (e.g., Bramhall SK7, not Bramhall elsewhere)
  let formattedLocation = location;
  if (postcode) {
    formattedLocation = `${location}, ${postcode}`;
  }
  
  // Ensure CUSTOM> prefix as required by HasData
  if (!formattedLocation.startsWith("CUSTOM>")) {
    formattedLocation = `CUSTOM>${formattedLocation}`;
  }
  
  // Default keywords if none provided
  const keywords = businessTypes.length > 0 ? businessTypes : ["businesses", "shops"];
  
  return new Promise((resolve, reject) => {
    const postData = JSON.stringify({
      keywords: keywords,
      locations: [formattedLocation],
      extractEmails: extractEmails
    });
    
    const options = {
      hostname: HASDATA_BASE_URL,
    

function fetchHasDataJson(jsonUrl, postcode, businessTypes = []) {
  return new Promise((resolve, reject) => {
    const url = new URL(jsonUrl);
    const options = {
      hostname: url.hostname,
      path: url.pathname + (url.search || ""),
      method: "GET",
      headers: {
        "Accept": "application/json"
      }
    };

    const req = https.request(options, (res) => {
      // Use Buffer pattern to prevent memory leak from string concatenation
      const chunks = [];
      let totalLength = 0;

      res.on("data", (chunk) => {
        chunks.push(chunk);
        totalLength += chunk.length;
      });

      res.on("end", () => {
        try {
          const data = Buffer.concat(chunks, totalLength).toString('utf8');
          const businesses = JSON.parse(data);
          const parsed = parseBusinesses(Array.isArray(businesses) ? businesses : [businesses], businessTypes);
          resolve(filterByPostcode(parsed, postcode));
        } catch (error) {
          const d

function pollHasDataJob(jobId, apiKey, postcode, businessTypes = []) {
  return new Promise((resolve, reject) => {
    let attempts = 0;

    const poll = () => {
      attempts++;

      const options = {
        hostname: HASDATA_BASE_URL,
        path: `/scrapers/jobs/${jobId}`,
        method: "GET",
        headers: {
          "x-api-key": apiKey,
          "Accept": "application/json"
        }
      };

      const req = https.request(options, (res) => {
        // Use Buffer pattern to prevent memory leak from string concatenation
        const chunks = [];
        let totalLength = 0;

        res.on("data", (chunk) => {
          chunks.push(chunk);
          totalLength += chunk.length;
        });

        res.on("end", () => {
          try {
            const data = Buffer.concat(chunks, totalLength).toString('utf8');
            const result = JSON.parse(data);

            if ((result.status === "finished" || result.status === "completed") && result.data) {
           

function filterByPostcode(businesses, postcode) {
  if (!postcode) {
    return businesses; // No postcode filter, return all
  }
  
  // Extract postcode prefix (first part before space, e.g., "SK7" from "SK7 1AA")
  const postcodePrefix = postcode.toUpperCase().split(" ")[0].trim();
  
  return businesses.filter(business => {
    const businessPostcode = business.postcode || extractPostcodeFromAddress(business.address);
    
    if (!businessPostcode) {
      // If no postcode found, include it (might be valid but missing data)
      return true;
    }
    
    // Check if postcode starts with the prefix
    const businessPrefix = businessPostcode.toUpperCase().split(" ")[0].trim();
    return businessPrefix === postcodePrefix;
  });
}

function extractPostcodeFromAddress(address) {
  if (!address) return null;
  
  // UK postcode pattern: e.g., "SK7 1AA", "M1 1AA", "SW1A 1AA"
  const postcodePattern = /\b([A-Z]{1,2}\d{1,2}[A-Z]?\s?\d[A-Z]{2})\b/i;
  const match = address.match(postcodePattern);
  
  return match ? match[1].toUpperCase() : null;
}