Function bodies 338 total

function demonstrateSizeAdjustments() {
  console.log("📊 CURRENT APPROACH (Simple Linear):");
  console.log("ARV = Living Area × Price Per SqFt × Improvement Factor");
  console.log("Problem: Doesn't account for size economies/premiums");
  console.log();

  console.log("🏠 EXAMPLE SCENARIO:");
  const subjectArea = 2141;
  const avgCompArea = 1728;
  const pricePerSqft = 267;
  const impFactor = 1.03;

  console.log(`Subject: ${subjectArea} sqft`);
  console.log(`Average Comp: ${avgCompArea} sqft`);
  console.log(`Base Price/SqFt: $${pricePerSqft}`);
  console.log();

  // Current approach
  const currentARV = subjectArea * pricePerSqft * impFactor;
  console.log(`Current ARV: ${subjectArea} × $${pricePerSqft} × ${impFactor} = $${Math.round(currentARV).toLocaleString()}`);
  console.log();

  console.log("🔧 PROPOSED SIZE-ADJUSTED APPROACHES:");
  console.log("━".repeat(50));

  console.log("\n1️⃣ DIMINISHING RETURNS MODEL:");
  console.log("Larger homes have slightly lower $/sqft due t

  function calculateDiminishingReturns(area, basePrice, avgArea) {
    // Size factor: larger homes get slight discount, smaller get premium
    const sizeRatio = area / avgArea;
    const sizeFactor = Math.pow(sizeRatio, 0.85); // Power < 1 = diminishing returns
    const adjustedPrice = basePrice * sizeFactor;

    return {
      sizeFactor: sizeFactor,
      adjustedPrice: Math.round(adjustedPrice),
      arv: Math.round(area * adjustedPrice * impFactor)
    };
  }

  function calculateTieredPricing(area, basePrice) {
    let adjustedPrice = basePrice;

    if (area <= 1000) adjustedPrice = basePrice * 1.10;      // Small homes: 10% premium
    else if (area <= 1500) adjustedPrice = basePrice * 1.05; // Medium homes: 5% premium
    else if (area <= 2000) adjustedPrice = basePrice * 1.00; // Large homes: base price
    else if (area <= 2500) adjustedPrice = basePrice * 0.95; // XL homes: 5% discount
    else adjustedPrice = basePrice * 0.90;                   // XXL homes: 10% discount

    return {
      tier: getTier(area),
      adjustedPrice: Math.round(adjustedPrice),
      arv: Math.round(area * adjustedPrice * impFactor)
    };
  }

  function getTier(area) {
    if (area <= 1000) return "Small (≤1000 sqft)";
    if (area <= 1500) return "Medium (1001-1500 sqft)";
    if (area <= 2000) return "Large (1501-2000 sqft)";
    if (area <= 2500) return "XL (2001-2500 sqft)";
    return "XXL (>2500 sqft)";
  }

  function calculateMarketBasedAdjustment(area, basePrice, avgArea) {
    // Simulate market data showing how price/sqft varies by size
    const sizeDeviation = (area - avgArea) / avgArea;

    // Market research shows: every 25% size increase = 2% price/sqft decrease
    const priceAdjustment = -sizeDeviation * 0.08; // 8% adjustment per 100% size change
    const sizeFactor = 1 + priceAdjustment;

    return {
      sizeDeviation: sizeDeviation,
      priceAdjustment: priceAdjustment,
      sizeFactor: sizeFactor,
      adjustedPrice: Math.round(basePrice * sizeFactor),
      arv: Math.round(area * basePrice * sizeFactor * impFactor)
    };
  }

function calculateSizeAdjustedPrice(livingArea, basePricePerSqft, avgCompArea) {
  // Diminishing returns: power factor < 1.0
  const sizeRatio = livingArea / avgCompArea;
  const sizeFactor = Math.pow(sizeRatio, 0.85); // 0.85 = moderate adjustment
  return Math.round(basePricePerSqft * sizeFactor);
}

async function demoSizeAdjustment() {
  console.log("🎯 SIMULATING REAL SCENARIO:");
  console.log("Large property (2,141 sqft) vs smaller average comps (1,728 sqft)");
  console.log();

  // Test property from the original issue
  const testProperty = {
    zpid: "demo_size",
    price: 499000,
    livingArea: 2141,
    bedrooms: 5,
    bathrooms: 3
  };

  // Create mock comparable data that matches the real structure
  const mockFilteredComps = [
    {
      price: { value: 420000 },
      sqFt: { value: 1600 },
      beds: 3
    },
    {
      price: { value: 450000 },
      sqFt: { value: 1700 },
      beds: 3
    },
    {
      price: { value: 480000 },
      sqFt: { value: 1800 },
      beds: 4
    },
    {
      price: { value: 510000 },
      sqFt: { value: 1750 },
      beds: 4
    },
    {
      price: { value: 490000 },
      sqFt: { value: 1820 },
      beds: 4
    }
  ];

  const avgCompSize = mockFilteredComps.reduce((sum, c) => sum + c.sqFt.value, 0) / mockFilteredComps.

function explainAddOnARV() {
  console.log("🏠 ADD-ON STRATEGY OVERVIEW:");
  console.log("Add-On adds a bedroom (typically 120 sqft) to existing property");
  console.log("ARV = Current Property Value + Addition Value + Extra Value");
  console.log();

  console.log("🔢 ADD-ON ARV FORMULA:");
  console.log("━".repeat(50));
  console.log("1. Current Property Value = Existing Area × Market Price/SqFt × Improvement Factor");
  console.log("2. Addition Value = Addition Area × Avg Comp Price/SqFt × Improvement Factor");
  console.log("3. Extra Value = Bedroom premium (if applicable)");
  console.log("4. Total ARV = Current Value + Addition Value + Extra Value");
  console.log();

  console.log("⚙️ ADD-ON CONFIGURATION:");
  console.log("━".repeat(50));
  console.log("• Addition Area: 120 sqft (bedroom size)");
  console.log("• Improvement Factor: 1.0 (no renovation premium)");
  console.log("• Duration: 6 months (default)");
  console.log("• Extra Value: Usually $0, can be bedroom premium");

function explainModifiedZScore() {
  console.log("🧮 WHAT IS THE MODIFIED Z-SCORE TEST?");
  console.log("━".repeat(50));
  console.log("The Modified Z-Score test is a statistical method for detecting outliers");
  console.log("that is MORE ROBUST than the traditional Z-Score because it uses:");
  console.log("• MEDIAN instead of MEAN (less affected by outliers)");
  console.log("• MAD instead of Standard Deviation (more resistant to extreme values)");
  console.log();

  console.log("📊 TRADITIONAL Z-SCORE vs MODIFIED Z-SCORE");
  console.log("━".repeat(50));
  console.log("Traditional Z-Score:");
  console.log("  Z = (value - mean) / standard_deviation");
  console.log("  ❌ Problem: Mean and StdDev are skewed by outliers");
  console.log();
  console.log("Modified Z-Score:");
  console.log("  Modified Z = 0.6745 × (value - median) / MAD");
  console.log("  ✅ Benefit: Median and MAD are outlier-resistant");
  console.log();

  console.log("🔢 STEP-BY-STEP CALCULATION");
  console.log("━"

async function createFluidCMProject(dealData, dealId) {
  const apiUrl = process.env.FLUIDCM_API_URL;
  const apiToken = process.env.FLUIDCM_API_TOKEN;
  const orgId = parseInt(process.env.FLUIDCM_ORG_ID || "1", 10);

  if (!apiUrl || !apiToken) {
    console.warn("[FluidCM] FLUIDCM_API_URL or FLUIDCM_API_TOKEN not configured, skipping handoff");
    return null;
  }

  // Build project code from address (e.g., "123 Main St" -> "RD-123-MAIN")
  const address = dealData.address || dealData.streetAddress || "Unknown";
  const projectCode = buildProjectCode(address);

  // Build project payload
  const payload = {
    code: projectCode,
    name: address,
    organization_id: orgId,
    description: buildDescription(dealData, dealId),
    zip_code: dealData.zipcode || dealData.zip || undefined,
    base_value: dealData.price || dealData.listPrice || undefined,
  };

  const response = await axios.post(
    `${apiUrl}/api/v1/projects`,
    payload,
    {
      headers: {
        "Authoriza

function buildProjectCode(address) {
  const parts = address.split(/[\s,]+/).filter(Boolean);
  const number = parts.find((p) => /^\d+$/.test(p)) || "";
  const street = parts.find((p) => /^[A-Za-z]{3,}$/.test(p) && !isStopWord(p)) || "";
  const code = `RD-${number}-${street}`.toUpperCase().slice(0, 15);
  return code || `RD-${Date.now().toString(36).toUpperCase()}`;
}

function isStopWord(word) {
  return ["the", "and", "ave", "street", "drive", "road", "lane", "way", "blvd", "court"].includes(
    word.toLowerCase(),
  );
}

function buildDescription(dealData, dealId) {
  const lines = [`Imported from RealDeal deal ${dealId}`];
  if (dealData.method) lines.push(`Strategy: ${dealData.method}`);
  if (dealData.price) lines.push(`List price: $${Number(dealData.price).toLocaleString()}`);
  if (dealData.score) lines.push(`Deal score: ${dealData.score}`);
  if (dealData.recommendation) lines.push(`Recommendation: ${dealData.recommendation}`);
  return lines.join("\n");
}

function initializeGA4Client(keyFilePath) {
  if (analyticsDataClient) {
    return analyticsDataClient;
  }

  analyticsDataClient = new BetaAnalyticsDataClient({
    keyFilename: keyFilePath,
  });

  return analyticsDataClient;
}

async function fetchLandingPageEvents(propertyId, startDate = 'yesterday', endDate = 'yesterday') {
  if (!analyticsDataClient) {
    throw new Error('GA4 client not initialized. Call initializeGA4Client() first.');
  }

  try {
    const [response] = await analyticsDataClient.runReport({
      property: `properties/${propertyId}`,
      dateRanges: [
        {
          startDate,
          endDate,
        },
      ],
      dimensions: [
        { name: 'eventName' },        // Event type (page_view, user_engagement, etc.)
        { name: 'date' },              // Date (YYYYMMDD format)
        { name: 'pagePath' },          // URL path
        { name: 'sessionSource' },     // Traffic source (google, facebook, direct, etc.)
        { name: 'sessionMedium' },     // Traffic medium (organic, cpc, referral, etc.)
      ],
      metrics: [
        { name: 'eventCount' },        // Number of events
        { name: 'activeUsers' },       // Number of unique users
        { name: 'averageS

async function fetchConversionEvents(propertyId, startDate = 'yesterday', endDate = 'yesterday') {
  if (!analyticsDataClient) {
    throw new Error('GA4 client not initialized. Call initializeGA4Client() first.');
  }

  try {
    const [response] = await analyticsDataClient.runReport({
      property: `properties/${propertyId}`,
      dateRanges: [{ startDate, endDate }],
      dimensions: [
        { name: 'eventName' },
        { name: 'date' },
        { name: 'sessionSource' },
        { name: 'sessionMedium' },
      ],
      metrics: [
        { name: 'eventCount' },
        { name: 'activeUsers' },
      ],
      // Only conversion events
      dimensionFilter: {
        filter: {
          fieldName: 'eventName',
          inListFilter: {
            values: [
              'form_submit',
              'sign_up',
              'generate_lead',
              'purchase',
              'contact',
            ],
          },
        },
      },
    });

    return parseGA4Respons

function parseGA4Response(response) {
  if (!response || !response.rows) {
    return [];
  }

  const events = [];

  response.rows.forEach((row) => {
    const event = {};

    // Parse dimensions
    row.dimensionValues.forEach((dimension, index) => {
      const dimensionName = response.dimensionHeaders[index].name;
      event[dimensionName] = dimension.value;
    });

    // Parse metrics
    row.metricValues.forEach((metric, index) => {
      const metricName = response.metricHeaders[index].name;
      event[metricName] = parseFloat(metric.value) || 0;
    });

    events.push(event);
  });

  return events;
}

async function getRealTimeMetrics(propertyId) {
  if (!analyticsDataClient) {
    throw new Error('GA4 client not initialized. Call initializeGA4Client() first.');
  }

  try {
    const [response] = await analyticsDataClient.runRealtimeReport({
      property: `properties/${propertyId}`,
      dimensions: [
        { name: 'eventName' },
        { name: 'country' },
      ],
      metrics: [
        { name: 'activeUsers' },
      ],
    });

    return parseGA4Response(response);
  } catch (error) {
    console.error('Error fetching real-time metrics:', error);
    throw error;
  }
}

async function testConnection(propertyId) {
  try {
    const [response] = await analyticsDataClient.runReport({
      property: `properties/${propertyId}`,
      dateRanges: [{ startDate: 'yesterday', endDate: 'yesterday' }],
      dimensions: [{ name: 'date' }],
      metrics: [{ name: 'activeUsers' }],
    });

    return {
      success: true,
      propertyId,
      rowCount: response.rowCount || 0,
      message: `Successfully connected to GA4 property ${propertyId}`,
    };
  } catch (error) {
    return {
      success: false,
      propertyId,
      error: error.message,
      message: 'Failed to connect to GA4',
    };
  }
}

function mapGA4Source(source, medium) {
  const lowerSource = (source || '').toLowerCase();
  const lowerMedium = (medium || '').toLowerCase();

  // Google Ads
  if (lowerSource.includes('google') && lowerMedium.includes('cpc')) {
    return 'google_ads';
  }

  // Facebook Ads
  if (lowerSource.includes('facebook') || lowerSource.includes('fb')) {
    if (lowerMedium.includes('cpc') || lowerMedium.includes('paid')) {
      return 'facebook_ads';
    }
    return 'facebook_organic';
  }

  // LinkedIn
  if (lowerSource.includes('linkedin')) {
    if (lowerMedium.includes('cpc') || lowerMedium.includes('paid')) {
      return 'linkedin_ads';
    }
    return 'linkedin_organic';
  }

  // Organic search
  if (lowerMedium === 'organic') {
    if (lowerSource.includes('google')) return 'google_organic';
    if (lowerSource.includes('bing')) return 'bing';
    return 'organic_search';
  }

  // Email
  if (lowerMedium === 'email' || lowerSource.includes('email')) {
    return 'email';
  }

function getFunnelStage(eventType) {
  const awarenessEvents = [
    'landing_page_view',
    'social_post',
    'ad_impression',
    'ad_click',
    'content_engagement',
  ];

  const followUpEvents = [
    'email_open',
    'email_click',
    'form_submission',
    'phone_call',
    'meeting_scheduled',
    'app_signup',
    'initial_contact',
  ];

  const convertedEvents = [
    'deal_closed',
    'contract_signed',
    'payment_received',
  ];

  const recommendationEvents = [
    'referral_made',
    'testimonial_given',
    'repeat_purchase',
  ];

  if (awarenessEvents.includes(eventType)) return 'awareness';
  if (followUpEvents.includes(eventType)) return 'follow_up';
  if (convertedEvents.includes(eventType)) return 'converted';
  if (recommendationEvents.includes(eventType)) return 'recommendation';

  return 'awareness'; // Default
}

function transformGA4Event(ga4Event, userId) {
  // Map GA4 event name to engagement event type
  const eventType = GA4_EVENT_MAP[ga4Event.eventName] || 'content_engagement';

  // Map source/medium to engagement source
  const source = mapGA4Source(ga4Event.sessionSource, ga4Event.sessionMedium);

  // Determine funnel stage
  const stage = getFunnelStage(eventType);

  // Parse date (GA4 format: YYYYMMDD)
  const dateStr = ga4Event.date || '';
  const timestamp = dateStr.length === 8
    ? admin.firestore.Timestamp.fromDate(
        new Date(
          dateStr.substring(0, 4), // Year
          parseInt(dateStr.substring(4, 6)) - 1, // Month (0-indexed)
          dateStr.substring(6, 8) // Day
        )
      )
    : admin.firestore.Timestamp.now();

  // Build engagement event
  const engagement = {
    userId,
    eventType,
    source,
    stage,
    timestamp,
    metadata: {
      ga4_event_name: ga4Event.eventName,
      ga4_source: ga4Event.sessionSource || 'unknown',
      ga

function transformGA4Batch(ga4Events, userId) {
  return ga4Events.map((event) => transformGA4Event(event, userId));
}

function deduplicateEngagements(engagements) {
  const seen = new Set();
  const unique = [];

  engagements.forEach((engagement) => {
    // Create unique key based on userId, eventType, source, and date
    const dateStr = engagement.timestamp.toDate().toISOString().split('T')[0];
    const key = `${engagement.userId}_${engagement.eventType}_${engagement.source}_${dateStr}`;

    if (!seen.has(key)) {
      seen.add(key);
      unique.push(engagement);
    }
  });

  return unique;
}

function aggregatePageViews(engagements) {
  const aggregated = [];
  const pageViewMap = new Map();

  engagements.forEach((engagement) => {
    if (engagement.eventType === 'landing_page_view') {
      const dateStr = engagement.timestamp.toDate().toISOString().split('T')[0];
      const key = `${engagement.userId}_${engagement.source}_${dateStr}`;

      if (pageViewMap.has(key)) {
        // Increment event count
        const existing = pageViewMap.get(key);
        existing.metadata.event_count += engagement.metadata.event_count || 1;
        existing.metadata.active_users = Math.max(
          existing.metadata.active_users || 0,
          engagement.metadata.active_users || 0
        );
      } else {
        pageViewMap.set(key, engagement);
      }
    } else {
      // Keep non-page-view events as-is
      aggregated.push(engagement);
    }
  });

  // Add aggregated page views
  pageViewMap.forEach((engagement) => aggregated.push(engagement));

  return aggregated;
}

function filterLowValueEvents(engagements) {
  return engagements.filter((engagement) => {
    // Keep conversion events always
    if (engagement.stage === 'converted') return true;

    // Keep follow-up events always
    if (engagement.stage === 'follow_up') return true;

    // For awareness events, filter out low engagement
    if (engagement.eventType === 'content_engagement') {
      const engagementRate = engagement.metadata.engagement_rate || 0;
      return engagementRate > 0.1; // Only keep if >10% engagement rate
    }

    // Keep everything else
    return true;
  });
}

function getHubSpotClient() {
  const apiKey = process.env.HUBSPOT_API_KEY;
  if (!apiKey) {
    throw new Error('HUBSPOT_API_KEY not set. Run: firebase functions:secrets:set HUBSPOT_API_KEY');
  }
  console.log('🔑 HubSpot API key found, length:', apiKey.length);
  return new hubspot.Client({ accessToken: apiKey });
}

async function createOrUpdateContact(contactData) {
  const hubspotClient = getHubSpotClient();

  const properties = {
    email: contactData.email,
  };

  if (contactData.firstName) properties.firstname = contactData.firstName;
  if (contactData.lastName) properties.lastname = contactData.lastName;
  if (contactData.phone) properties.phone = contactData.phone;

  // Add any custom properties
  if (contactData.customProperties) {
    Object.assign(properties, contactData.customProperties);
  }

  try {
    // Try to create the contact
    const response = await hubspotClient.crm.contacts.basicApi.create({
      properties,
      associations: []
    });
    console.log('✅ Contact created in HubSpot:', response.id);
    return response;
  } catch (error) {
    // If contact already exists (409 conflict), update it
    if (error.statusCode === 409) {
      console.log('Contact already exists, updating...');
      const existingContactId = error.body.message.match(/Existing ID: (\d+)/)?

async function createDeal(dealData) {
  const hubspotClient = getHubSpotClient();

  const properties = {
    dealname: dealData.dealName,
  };

  if (dealData.dealStage) properties.dealstage = dealData.dealStage;
  if (dealData.amount) properties.amount = dealData.amount.toString();
  if (dealData.pipeline) properties.pipeline = dealData.pipeline;
  if (dealData.closeDate) properties.closedate = dealData.closeDate;

  // Add any custom properties
  if (dealData.customProperties) {
    Object.assign(properties, dealData.customProperties);
  }

  const associations = [];

  // Associate with contact if provided
  if (dealData.contactId) {
    associations.push({
      to: { id: dealData.contactId },
      types: [{
        associationCategory: 'HUBSPOT_DEFINED',
        associationTypeId: 3 // Deal to Contact association
      }]
    });
  }

  const response = await hubspotClient.crm.deals.basicApi.create({
    properties,
    associations
  });

  console.log('✅ Deal created in HubSpo

async function createPropertyNote(noteData) {
  const hubspotClient = getHubSpotClient();

  const noteBody = `
📍 Property Analysis: ${noteData.propertyAddress || 'N/A'}

Strategy: ${noteData.strategy || 'N/A'}

Generated by CloudCalcs on ${new Date().toLocaleString()}
`;

  const properties = {
    hs_timestamp: Date.now().toString(),
    hs_note_body: noteBody,
  };

  const associations = [];

  if (noteData.contactId) {
    associations.push({
      to: { id: noteData.contactId },
      types: [{
        associationCategory: 'HUBSPOT_DEFINED',
        associationTypeId: 202 // Note to Contact association
      }]
    });
  }

  const response = await hubspotClient.crm.objects.notes.basicApi.create({
    properties,
    associations
  });

  console.log('✅ Property note created in HubSpot:', response.id);
  return response;
}

async function findContactByEmail(email) {
  const hubspotClient = getHubSpotClient();

  try {
    const response = await hubspotClient.crm.contacts.searchApi.doSearch({
      filterGroups: [{
        filters: [{
          propertyName: 'email',
          operator: 'EQ',
          value: email
        }]
      }],
      properties: ['email', 'firstname', 'lastname', 'phone'],
      limit: 1
    });

    if (response.results && response.results.length > 0) {
      console.log('✅ Contact found in HubSpot:', response.results[0].id);
      return response.results[0];
    }

    console.log('ℹ️ Contact not found in HubSpot for email:', email);
    return null;
  } catch (error) {
    console.error('Error searching for contact:', error);
    throw error;
  }
}

async function trackPropertyCalculation(eventData) {
  try {
    // Find or create contact
    let contact = await findContactByEmail(eventData.email);

    if (!contact) {
      contact = await createOrUpdateContact({
        email: eventData.email,
        firstName: eventData.firstName,
        lastName: eventData.lastName,
        phone: eventData.phone
      });
    }

    // Create note with calculation details (only address and method)
    const note = await createPropertyNote({
      contactId: contact.id,
      propertyAddress: eventData.address,
      strategy: eventData.method
    });

    return {
      success: true,
      contact: contact,
      note: note
    };
  } catch (error) {
    console.error('Error tracking property calculation:', error);
    throw error;
  }
}

function ensureFlutterFlowCompatibility(obj) {
  if (obj === null || obj === undefined) {
    return {error: "Null object"};
  }
  // Convert to JSON and back to remove any problematic values
  try {
    const jsonString = JSON.stringify(obj, (key, value) => {
      if (value === null || value === undefined) return "";
      if (typeof value === "number" && (isNaN(value) || !isFinite(value))) return 0;
      return value;
    });
    return JSON.parse(jsonString);
  } catch (error) {
    return {error: "JSON serialization failed", originalError: error.message};
  }
}

async function processBatch(batch, params, writeEvent, batchStartCount, totalProcessed, maxProperties) {
  console.log(`🚀 Processing batch of ${batch.length} properties (total processed: ${totalProcessed}/${maxProperties})`);

  writeEvent("status", {
    message: `Processing batch of ${batch.length} properties...`,
    totalProcessed: totalProcessed,
    maxProperties: maxProperties,
    timestamp: new Date().toISOString()
  });

  // Process all properties in the batch in parallel
  const batchPromises = batch.map(async (item, index) => {
    try {
      console.log(`🏠 Processing property ${item.property.zpid} (${item.sequence}/${item.totalEstimated})`);
      const results = await processProperty(item.property, params, item.sequence, item.totalEstimated);
      console.log(`✅ Property ${item.property.zpid} completed, got ${results.length} results`);
      return results;
    } catch (error) {
      console.error(`❌ Error processing property ${item.property.zpid}:`, {
        error: 

async function processUserDripCampaigns(userId, db) {
  let emailsSent = 0;

  // Get all contacts for this user (unique emails)
  const contactsMap = await getUserContacts(userId, db);

  // Check each contact for drip campaign eligibility
  for (const [email, lastEngagement] of contactsMap.entries()) {
    // Check 7-day follow-up
    if (shouldSend7DayFollowUp(lastEngagement)) {
      await send7DayEmail(userId, email, lastEngagement, db);
      emailsSent++;
    }
    // Check 30-day check-in
    else if (shouldSend30DayCheckIn(lastEngagement)) {
      await send30DayEmail(userId, email, lastEngagement, db);
      emailsSent++;
    }
  }

  return emailsSent;
}

async function getUserContacts(userId, db) {
  const snapshot = await db
    .collection("engagements")
    .where("userId", "==", userId)
    .where("contactEmail", "!=", null)
    .orderBy("contactEmail")
    .orderBy("timestamp", "desc")
    .get();

  const contactsMap = new Map();

  snapshot.docs.forEach((doc) => {
    const data = doc.data();
    const email = data.contactEmail;

    // Only keep the most recent engagement per contact
    if (email && !contactsMap.has(email)) {
      contactsMap.set(email, {
        ...data,
        id: doc.id,
      });
    }
  });

  return contactsMap;
}

function shouldSend7DayFollowUp(engagement) {
  const now = new Date();
  const engagementDate = engagement.timestamp.toDate();
  const daysSince = Math.floor(
    (now.getTime() - engagementDate.getTime()) / (1000 * 60 * 60 * 24)
  );

  // Send if:
  // 1. Last engagement was 7 days ago
  // 2. Last engagement was initial_contact or email_sent
  // 3. Haven't sent 7-day follow-up yet

  if (daysSince !== 7) {
    return false;
  }

  if (
    engagement.eventType !== "initial_contact" &&
    engagement.eventType !== "email_sent"
  ) {
    return false;
  }

  // Check if we've already sent 7-day follow-up
  if (engagement.metadata && engagement.metadata.drip7DaySent) {
    return false;
  }

  return true;
}

function shouldSend30DayCheckIn(engagement) {
  const now = new Date();
  const engagementDate = engagement.timestamp.toDate();
  const daysSince = Math.floor(
    (now.getTime() - engagementDate.getTime()) / (1000 * 60 * 60 * 24)
  );

  // Send if:
  // 1. Last engagement was 30 days ago
  // 2. Last engagement was not converted or recommendation stage
  // 3. Haven't sent 30-day check-in yet

  if (daysSince !== 30) {
    return false;
  }

  if (
    engagement.eventType === "deal_closed" ||
    engagement.eventType === "contract_signed" ||
    engagement.eventType === "payment_received" ||
    engagement.eventType === "referral_made" ||
    engagement.eventType === "testimonial_given"
  ) {
    return false;
  }

  // Check if we've already sent 30-day check-in
  if (engagement.metadata && engagement.metadata.drip30DaySent) {
    return false;
  }

  return true;
}

async function send7DayEmail(userId, email, engagement, db) {
  console.log(`[Drip] Sending 7-day follow-up to ${email}`);

  // Create email document in 'mail' collection
  await db.collection("mail").add({
    to: email,
    template: {
      name: "realdeal-followup-7day",
      data: {
        contactName: email.split("@")[0], // Extract name from email
      },
    },
  });

  // Mark as sent in engagement metadata
  await db.collection("engagements").doc(engagement.id).update({
    "metadata.drip7DaySent": true,
    "metadata.drip7DaySentAt": admin.firestore.FieldValue.serverTimestamp(),
  });

  // Create engagement event
  await db.collection("engagements").add({
    eventType: "email_sent",
    source: "email",
    contactEmail: email,
    userId,
    timestamp: admin.firestore.FieldValue.serverTimestamp(),
    metadata: {
      dripCampaign: "7-day-followup",
      templateId: "realdeal-followup-7day",
    },
  });
}

async function send30DayEmail(userId, email, engagement, db) {
  console.log(`[Drip] Sending 30-day check-in to ${email}`);

  // Create email document in 'mail' collection
  await db.collection("mail").add({
    to: email,
    template: {
      name: "realdeal-checkin-30day",
      data: {
        contactName: email.split("@")[0],
      },
    },
  });

  // Mark as sent in engagement metadata
  await db.collection("engagements").doc(engagement.id).update({
    "metadata.drip30DaySent": true,
    "metadata.drip30DaySentAt": admin.firestore.FieldValue.serverTimestamp(),
  });

  // Create engagement event
  await db.collection("engagements").add({
    eventType: "email_sent",
    source: "email",
    contactEmail: email,
    userId,
    timestamp: admin.firestore.FieldValue.serverTimestamp(),
    metadata: {
      dripCampaign: "30-day-checkin",
      templateId: "realdeal-checkin-30day",
    },
  });
}

async function loadUserCalcParams(db, email) {
  const userSnap = await db.collection("UserData")
    .where("email", "==", email).limit(1).get();

  if (userSnap.empty) {
    console.log(`[DealScanner] No user profile found for ${email}, using defaults`);
    return {};
  }

  const u = userSnap.docs[0].data();
  console.log(`[DealScanner] Loaded profile for ${email} (${u.display_name || "unknown"})`);

  // Map user profile field names → strategyCalculator param names
  const params = {};
  if (u.financingRate) params.interestRate = u.financingRate;
  if (u.salRate) params.salRate = u.salRate;
  if (u.loanFeesRate) params.loanFeesRate = u.loanFeesRate;
  if (u.permitsFees) params.permitsFees = u.permitsFees;
  if (u.fixnflipDuration) params.fixFlipDuration = u.fixnflipDuration;
  if (u.addOnDuration) params.addOnDuration = u.addOnDuration;
  if (u.aduDuration) params.aduDuration = u.aduDuration;
  if (u.newDuration) params.newBuildDuration = u.newDuration;
  if (u.oneBdrmMarketValue)

async function fetchZillowDataWithCache(endpoint, config, maxRetries = 2) {
  const docId = `oa_zillow_${endpoint}_${Buffer.from(JSON.stringify(config)).toString("base64")}`;

  async function realFetch() {
    let retries = 0;
    while (retries <= maxRetries) {
      try {
        let url;
        let params = {};

        if (endpoint === "zestimate") {
          // zpid is "oa_<id>" — extract address from the prop context
          // For zestimate, we need address+city. The caller passes {zpid}.
          // Since OA uses address-based lookup, we pass zpid as-is and
          // the API will resolve it. If zpid starts with "oa_", use the
          // listing search to find it.
          if (config.address && config.city) {
            url = `${OA_DATA_API_URL}/api/v1/valuation/estimate`;
            params = { address: config.address, city: config.city, state: config.state || "CA" };
          } else {
            // Fallback: return empty data (no zpid-based lookup in OA)
       

  async function realFetch() {
    let retries = 0;
    while (retries <= maxRetries) {
      try {
        let url;
        let params = {};

        if (endpoint === "zestimate") {
          // zpid is "oa_<id>" — extract address from the prop context
          // For zestimate, we need address+city. The caller passes {zpid}.
          // Since OA uses address-based lookup, we pass zpid as-is and
          // the API will resolve it. If zpid starts with "oa_", use the
          // listing search to find it.
          if (config.address && config.city) {
            url = `${OA_DATA_API_URL}/api/v1/valuation/estimate`;
            params = { address: config.address, city: config.city, state: config.state || "CA" };
          } else {
            // Fallback: return empty data (no zpid-based lookup in OA)
            return { status: 200, data: { value: 0 } };
          }
        } else if (endpoint === "propertyDetails") {
          if (config.address && config.city) {
            url

async function fetchRedfinDataWithCache(endpoint, config, maxRetries = 2) {
  const docId = `oa_redfin_${endpoint}_${Buffer.from(JSON.stringify(config)).toString("base64")}`;

  async function realFetch() {
    let retries = 0;
    while (retries <= maxRetries) {
      try {
        let url;
        const params = { location: config.location, limit: 50 };

        if (endpoint === "searchSold") {
          url = `${OA_DATA_API_URL}/api/v1/comps/sold`;
        } else if (endpoint === "searchForSale") {
          url = `${OA_DATA_API_URL}/api/v1/comps/for-sale`;
        } else {
          console.warn(`[OA-API] Unknown Redfin endpoint: ${endpoint}`);
          return { status: 200, data: { data: { homes: [] } } };
        }

        const response = await axios.get(url, { params, timeout: 10000 });

        // OA returns {status, data: {data: {homes: [...]}}} — matches Redfin shape
        return { status: 200, data: response.data.data || { data: { homes: [] } } };
      } catch (err) {

  async function realFetch() {
    let retries = 0;
    while (retries <= maxRetries) {
      try {
        let url;
        const params = { location: config.location, limit: 50 };

        if (endpoint === "searchSold") {
          url = `${OA_DATA_API_URL}/api/v1/comps/sold`;
        } else if (endpoint === "searchForSale") {
          url = `${OA_DATA_API_URL}/api/v1/comps/for-sale`;
        } else {
          console.warn(`[OA-API] Unknown Redfin endpoint: ${endpoint}`);
          return { status: 200, data: { data: { homes: [] } } };
        }

        const response = await axios.get(url, { params, timeout: 10000 });

        // OA returns {status, data: {data: {homes: [...]}}} — matches Redfin shape
        return { status: 200, data: response.data.data || { data: { homes: [] } } };
      } catch (err) {
        if (err.response?.status === 429 || err.code === "ECONNREFUSED") {
          await new Promise((r) => setTimeout(r, 1000 + retries * 500));
          retries++;
       

async function fetchMarketSignal(zipCode) {
  try {
    const response = await axios.get(`${OA_DATA_API_URL}/api/v1/market/signal`, {
      params: { zip_code: zipCode },
      timeout: 5000,
    });
    return response.data;
  } catch (err) {
    console.warn(`[OA-API] Market signal error: ${err.message}`);
    return { signal: "yellow", composite_score: 50, recommendation: "Data unavailable" };
  }
}

async function fetchDistressCheck(address, city) {
  try {
    const response = await axios.get(`${OA_DATA_API_URL}/api/v1/distress/check`, {
      params: { address, city },
      timeout: 5000,
    });
    return response.data;
  } catch (err) {
    console.warn(`[OA-API] Distress check error: ${err.message}`);
    return {
      has_distress: false,
      is_foreclosure: false,
      is_pre_foreclosure: false,
      has_tax_delinquency: false,
    };
  }
}

  function extractCompPrice(c) {
    return c.data?.aboveTheFold?.addressSectionInfo?.priceInfo?.amount ||
      (typeof c.price === "object" ? c.price?.value : c.price) || 0;
  }

  function extractCompSqft(c) {
    return c.data?.aboveTheFold?.addressSectionInfo?.sqFt?.value ||
      (typeof c.sqFt === "object" ? c.sqFt?.value : c.sqFt) || 0;
  }

  function extractLotSize(c) {
    // If lotSize is an object with value property, extract it
    if (typeof c.lotSize === "object" && c.lotSize !== null) {
      return c.lotSize.value || 0;
    }
    return c.lotSize || 0;
  }