server.js

gistfile1.txt

import express from 'express';
import cors from 'cors';
import path from 'path';
import { fileURLToPath } from 'url';
import fs from 'fs';
import { chromium } from 'playwright';

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);

const app = express();
app.use(cors());
app.use(express.json());
app.use(express.static('public'));

// Create logs directory if it doesn't exist
const logsDir = './logs';
if (!fs.existsSync(logsDir)) {
  fs.mkdirSync(logsDir, { recursive: true });
  console.log('📁 Created logs directory');
}

// Function to format question for cleaner Web UI responses
function formatQuestionForWebUI(originalQuestion) {
  return `IMPORTANT: Answer in clean raw markdown without any contentReference citations or oaicite references. Do not include any :contentReference[oaicite:X]{index=X} patterns in your response.
  Original question: ${originalQuestion}
- Use \`\`\` for code blocks
 Wrapp the entire response in a markdown  block to show the actual syntax.
 Do not disclose any informatin about the owner of this chargpt account.
 `;
}

// Function to clean Web UI response content
function cleanWebUIResponse(rawText) {
  if (!rawText) return rawText;
  
  let cleaned = rawText
    // Remove contentReference citations
    .replace(/:contentReference\[oaicite:\d+\]\{index=\d+\}/g, '')
    .replace(/contentReference\[oaicite:\d+\]\{index=\d+\}/g, '')
    .replace(/:contentReference\[[^\]]+\]/g, '')
    .replace(/contentReference\[[^\]]+\]/g, '')
    // Remove oaicite patterns
    .replace(/oaicite:\d+/g, '')
    .replace(/\{index=\d+\}/g, '')
    // Remove copy/edit buttons
    .replace(/markdownCopyEdit/g, '')
    .replace(/javascriptCopyEdit/g, '')
    .replace(/CopyEdit/g, '')
    .replace(/Copy/g, '')
    .replace(/Edit/g, '')
    // Clean up formatting
    .replace(/🔹\s*/g, '\n\n')
    .replace(/\n{3,}/g, '\n\n')
    // Remove any remaining citation artifacts
    .replace(/\[\d+\]/g, '')
    .replace(/\(\d+\)/g, '')
    .trim();
    
  return cleaned;
}

// Logging function
function logResponse(type, question, response, error = null, metadata = {}) {
  try {
    const timestamp = new Date().toISOString();
    const logEntry = {
      timestamp,
      type,
      question,
      response,
      error,
      metadata,
      success: !error
    };
    
    const filename = `${timestamp.split('T')[0]}_${type}_responses.json`;
    const filepath = path.join(logsDir, filename);
    
    let logs = [];
    if (fs.existsSync(filepath)) {
      try {
        const existingData = fs.readFileSync(filepath, 'utf8');
        logs = JSON.parse(existingData);
      } catch (e) {
        console.log('📝 Creating new log file:', filename);
      }
    }
    
    logs.push(logEntry);
    fs.writeFileSync(filepath, JSON.stringify(logs, null, 2));
    console.log(`📝 Logged ${type} response to ${filename}`);
  } catch (e) {
    console.error('❌ Error writing log:', e.message);
  }
}

// Serve the main chat interface
app.get('/', (req, res) => {
  res.sendFile(path.join(__dirname, 'public', 'index.html'));
});

// Store session URL for maintaining conversation history
let currentSessionUrl = null;

// ChatGPT streaming endpoint
app.post('/api/chat', async (req, res) => {
  const { question, sessionUrl } = req.body;
  const headlessMode = req.body.headless !== false; // Default to true if not specified
  
  if (!question) {
    return res.status(400).json({ error: 'Question is required' });
  }

  // Update session URL if provided
  if (sessionUrl) {
    currentSessionUrl = sessionUrl;
    console.log('🔗 Session URL updated:', currentSessionUrl);
  }

  // Set up Server-Sent Events
  res.writeHead(200, {
    'Content-Type': 'text/event-stream',
    'Cache-Control': 'no-cache',
    'Connection': 'keep-alive',
    'Access-Control-Allow-Origin': '*',
    'Access-Control-Allow-Headers': 'Cache-Control'
  });

  // Send initial connection confirmation
  res.write(`data: ${JSON.stringify({ type: 'connected', message: 'Connected to server' })}\n\n`);

  // Process ChatGPT request
  await processChatGPTRequest(question, headlessMode, res, currentSessionUrl);
});

// Main ChatGPT processing function
async function processChatGPTRequest(question, headlessMode = true, res, sessionUrl = null) {
  console.log('🔍 DEBUG: Starting ChatGPT processing...');
  console.log('🔍 DEBUG: Question received:', question);
  console.log('🔍 DEBUG: Headless mode:', headlessMode);

  let browser = null;
  let context = null;
  let page = null;

  const sendEvent = (type, data) => {
    try {
      const eventData = JSON.stringify({ type, ...data });
      res.write(`data: ${eventData}\n\n`);
      console.log(`📡 Sent event: ${type}`, data);
    } catch (e) {
      console.error('❌ Error sending event:', e.message);
    }
  };

  const forceCleanup = async () => {
    console.log('🚨 Force cleanup initiated...');
    try {
      if (context) {
        console.log('🔒 Force closing context...');
        await Promise.race([
          context.close(),
          new Promise(resolve => setTimeout(resolve, 5000))
        ]);
        console.log('✅ Context force closed');
        context = null;
      }
    } catch (e) {
      console.error('❌ Error force closing context:', e.message);
    }
    
    try {
      if (browser) {
        console.log('🚪 Force closing browser...');
        await Promise.race([
          browser.close(),
          new Promise(resolve => setTimeout(resolve, 5000))
        ]);
        console.log('✅ Browser force closed');
        browser = null;
      }
    } catch (e) {
      console.error('❌ Error force closing browser:', e.message);
    }
  };

  const maxOperationTime = 180000; // 3 minutes total
  const operationTimeout = setTimeout(async () => {
    console.log('⏰ Operation timeout reached, forcing cleanup...');
    sendEvent('error', { message: 'Operation timeout - forcing browser closure' });
    await forceCleanup();
    try {
      res.end();
    } catch (e) {
      console.error('❌ Error ending response:', e.message);
    }
  }, maxOperationTime);

  try {
    console.log('🔍 DEBUG: Starting browser launch process...');
    const modeText = headlessMode ? 'headless' : 'head full (visible)';
    const modeIcon = headlessMode ? '🚀' : '👁️';
    res.write(`data: ${JSON.stringify({ type: 'status', message: 'launching_agent' })}\n\n`);
    
    // Launch browser with configurable headless mode
    console.log(`🔍 DEBUG: Calling chromium.launch() with headless=${headlessMode}...`);
    
    const launchArgs = [
      '--disable-blink-features=AutomationControlled',
      '--disable-dev-shm-usage',
      '--no-sandbox', // Required for cloud environments
      '--disable-setuid-sandbox',
      '--disable-web-security',
      '--disable-features=VizDisplayCompositor',
      '--no-first-run',
      '--disable-default-apps',
      '--disable-extensions',
      '--window-size=1280,720'  // required for proper headless operation
    ];
    
    // Configure new headless mode to fix ChatGPT WebGL/Canvas issues
    if (headlessMode) {
      launchArgs.push(
        '--headless=new',          // switch to Chrome's modern headless engine
        '--use-gl=swiftshader'     // software GL so WebGL still works
      );
    }
    
    browser = await chromium.launch({
      headless: headlessMode,      // strictly boolean!
      args: launchArgs,
      channel: 'chrome'            // optional, but guarantees a recent Chrome binary
    });
    console.log(`🔍 DEBUG: Browser launched successfully in ${modeText} mode`);

    // Use persistent context to save login session
    let contextOptions = {
      userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
      viewport: { width: 1280, height: 720 }
    };
    
    // Only load storage state if file exists
    try {
      if (fs.existsSync('./.puppeteer_profile/playwright_storage.json')) {
        contextOptions.storageState = './.puppeteer_profile/playwright_storage.json';
      } else {
        sendEvent('status', { message: '⚠️ No saved login session found. Please run "node playwright-login.js" first.' });
      }
    } catch (e) {
      sendEvent('status', { message: '⚠️ Could not check for storage file' });
    }
    
    console.log('🔍 DEBUG: Creating browser context...');
    context = await browser.newContext(contextOptions);
    console.log('🔍 DEBUG: Browser context created');
    
    console.log('🔍 DEBUG: Creating new page...');
    page = await context.newPage();
    console.log('🔍 DEBUG: New page created');

    // Add extra stealth
    console.log('🔍 DEBUG: Adding stealth scripts...');
    await page.addInitScript(() => {
      Object.defineProperty(navigator, 'webdriver', {
        get: () => undefined,
      });
      
      delete window.navigator.__proto__.webdriver;
      
      window.chrome = {
        runtime: {},
        loadTimes: function() {},
        csi: function() {},
        app: {}
      };
    });
    console.log('🔍 DEBUG: Stealth scripts added');

    console.log('🔍 DEBUG: Navigating Intel Database...');
    res.write(`data: ${JSON.stringify({ type: 'status', message: 'sending_to_engine' })}\n\n`);
    
    // Use session URL if available, otherwise use base URL
    const targetUrl = sessionUrl || 'https://chat.openai.com/';
    console.log('🔍 DEBUG: Target URL:', targetUrl);
    
    if (sessionUrl) {
      console.log('🔗 Using existing session URL to maintain conversation history');
      sendEvent('status', { message: 'Continuing existing conversation...' });
    } else {
      console.log('🆕 Starting new conversation');
      sendEvent('status', { message: 'Starting new conversation...' });
    }
    
    await page.goto(targetUrl, { waitUntil: 'networkidle' });
    console.log('🔍 DEBUG: Navigation completed');

    console.log('🔍 DEBUG: Waiting 1 second...');
    await page.waitForTimeout(1000);
    console.log('🔍 DEBUG: Wait completed');

    // Check if logged in
    console.log('🔍 DEBUG: Checking login status...');
    const textareaCount = await page.locator('textarea').count();
    console.log(`🔍 DEBUG: Found ${textareaCount} textarea elements`);
    
    const contentEditableCount = await page.locator('[contenteditable="true"]').count();
    console.log(`🔍 DEBUG: Found ${contentEditableCount} contenteditable elements`);
    
    const isLoggedIn = textareaCount > 0 || contentEditableCount > 0;
    console.log(`🔍 DEBUG: Login status: ${isLoggedIn ? 'LOGGED IN' : 'NOT LOGGED IN'}`);

    if (!isLoggedIn) {
      console.log('🔍 DEBUG: User not logged in, sending error...');
      
      // Save current state even if not logged in, so user can log in manually
      try {
        await context.storageState({ path: './.puppeteer_profile/playwright_storage.json' });
        console.log('💾 Current session state saved for manual login');
      } catch (e) {
        console.error('❌ Error saving storage state:', e.message);
      }
      
      sendEvent('error', { 
        message: 'Not logged in. Please run "node playwright-login.js" first to log in manually.',
        suggestion: 'After logging in once, the session will be saved and reused for both headless and visible modes.'
      });
      throw new Error('Not logged in. Please run the manual login first.');
    }

    console.log('🔍 DEBUG: User is logged in, proceeding with question...');
    res.write(`data: ${JSON.stringify({ type: 'status', message: 'sending_formatted_question' })}\n\n`);
    
    const formattedQuestion = formatQuestionForWebUI(question);
    console.log('🔍 DEBUG: Formatted question:', formattedQuestion.substring(0, 100) + '...');
    
    console.log('🔍 DEBUG: Looking for input elements...');
    const contentEditable = page.locator('[contenteditable="true"]').first();
    const contentEditableVisible = await contentEditable.isVisible().catch(() => false);
    console.log('🔍 DEBUG: ContentEditable visible:', contentEditableVisible);
    
    if (contentEditableVisible) {
      console.log('🔍 DEBUG: Using contenteditable input...');
      await contentEditable.click();
      console.log('🔍 DEBUG: Clicked contenteditable');
      await contentEditable.fill(formattedQuestion);
      console.log('🔍 DEBUG: Filled contenteditable with question');
    } else {
      console.log('🔍 DEBUG: Using textarea input...');
      await page.locator('textarea').first().click();
      console.log('🔍 DEBUG: Clicked textarea');
      await page.locator('textarea').first().fill(formattedQuestion);
      console.log('🔍 DEBUG: Filled textarea with question');
    }
    
    console.log('🔍 DEBUG: Pressing Enter to submit...');
    await page.keyboard.press('Enter');
    console.log('🔍 DEBUG: Enter pressed');

    // Monitor URL changes to catch session ID
    console.log('🔍 DEBUG: Setting up URL monitoring...');
    let sessionId = null;
    let initialUrl = page.url();
    console.log('🔍 DEBUG: Initial URL:', initialUrl);
    
    // Enhanced URL monitoring with multiple patterns and more frequent checks
    const checkUrlForSession = (url, source = 'unknown') => {
      console.log(`🔍 DEBUG: Checking URL from ${source}:`, url);
      
      // Try multiple URL patterns for ChatGPT
      const patterns = [
        /https:\/\/chatgpt\.com\/c\/([a-f0-9-]+)/i,
        /https:\/\/chat\.openai\.com\/c\/([a-f0-9-]+)/i,
        /https:\/\/chatgpt\.com\/chat\/([a-f0-9-]+)/i,
        /https:\/\/chat\.openai\.com\/chat\/([a-f0-9-]+)/i
      ];
      
      for (const pattern of patterns) {
        const match = url.match(pattern);
        if (match && match[1] && !sessionId) {
          sessionId = match[1];
          
          // Enhanced logging for session ID capture
          console.log('\n' + '='.repeat(80));
          console.log(`🎯 CHATGPT SESSION ID CAPTURED! (${source})`);
          console.log('📋 Session ID:', sessionId);
          console.log('🔗 Full Conversation URL:', url);
          console.log('⏰ Timestamp:', new Date().toISOString());
          console.log('❓ Original Question:', question.substring(0, 100) + (question.length > 100 ? '...' : ''));
          console.log('🔍 Pattern matched:', pattern.toString());
          console.log('='.repeat(80) + '\n');
          
          // Update global session URL for future requests
          currentSessionUrl = url;
          
          // Send session info to client
          sendEvent('session', { 
            sessionId: sessionId,
            conversationUrl: url,
            message: `Session ID captured: ${sessionId}`,
            timestamp: new Date().toISOString(),
            source: source
          });
          
          return true;
        }
      }
      return false;
    };
    
    // Set up URL change listener
    page.on('framenavigated', (frame) => {
      if (frame === page.mainFrame()) {
        const newUrl = frame.url();
        console.log('🔍 DEBUG: Frame navigated - URL changed to:', newUrl);
        checkUrlForSession(newUrl, 'framenavigated');
      }
    });
    
    // Also listen for URL changes via page events
    page.on('load', () => {
      const currentUrl = page.url();
      console.log('🔍 DEBUG: Page loaded - URL:', currentUrl);
      checkUrlForSession(currentUrl, 'page_load');
    });
    
    // More frequent URL checking with detailed logging
    const urlCheckInterval = setInterval(async () => {
      try {
        const currentUrl = page.url();
        
        if (currentUrl !== initialUrl && !sessionId) {
          console.log('🔍 DEBUG: URL has changed from initial, checking for session...');
          const found = checkUrlForSession(currentUrl, 'periodic_check');
          if (found) {
            clearInterval(urlCheckInterval);
          }
        }
      } catch (e) {
        console.log('🔍 DEBUG: Error in URL check interval:', e.message);
        clearInterval(urlCheckInterval);
      }
    }, 200); // Check every 200ms (more frequent)
    
    // Clear interval after 2 minutes to avoid infinite checking
    setTimeout(() => {
      if (urlCheckInterval) {
        clearInterval(urlCheckInterval);
        console.log('🔍 DEBUG: URL monitoring interval cleared after 2 minutes');
      }
    }, 120000);

    console.log('🔍 DEBUG: Waiting for response to start...');
    res.write(`data: ${JSON.stringify({ type: 'status', message: 'waiting_for_response' })}\n\n`);
    
    await page.waitForSelector('[data-message-author-role="assistant"]', { timeout: 30000 });
    console.log('🔍 DEBUG: Response element found');
    res.write(`data: ${JSON.stringify({ type: 'status', message: 'response_streaming_started' })}\n\n`);
    console.log('🔍 DEBUG: Status event sent, starting content loop...');
    
    let lastContent = '';
    let attempts = 0;
    let responseComplete = false;
    let finalContent = '';
    let noChangeCount = 0;
    const maxAttempts = 1200;  // 2 minutes at 100ms intervals
    const maxNoChangeAttempts = 15;
    
    while (attempts < maxAttempts && !responseComplete) {
      await page.waitForTimeout(100);
      attempts++;
      
      const stopButton = await page.locator('button[aria-label*="Stop"], button:has-text("Stop generating")').count();
      const isGenerating = stopButton > 0;
      
      const selectors = [
        '[data-message-author-role="assistant"] .markdown',
        '[data-message-author-role="assistant"] div[class*="markdown"]',
        '[data-message-author-role="assistant"] div',
        '[data-message-author-role="assistant"] p',
        '[data-message-author-role="assistant"]',
        '.markdown',
        '.prose',
        'div[data-message-id] div',
        'div[data-testid*="conversation"] div'
      ];
      
      let currentContent = null;
      for (const selector of selectors) {
        try {
          const elements = await page.locator(selector).all();
          if (attempts <= 5) { // Only log for first few attempts to reduce noise
            console.log(`🔍 DEBUG: Selector "${selector}" found ${elements.length} elements`);
          }
          if (elements.length > 0) {
            const lastElement = elements[elements.length - 1];
            
            // Try multiple methods to get text content
            let text = null;
            try {
              text = await lastElement.textContent();
              if (attempts <= 5) {
                console.log(`🔍 DEBUG: textContent from "${selector}": "${text ? text.substring(0, 100) : 'null'}"...`);
              }
            } catch (e) {
              if (attempts <= 5) {
                console.log(`� DEBUG: textContent failed for "${selector}":`, e.message);
              }
            }
            
            if (!text || !text.trim()) {
              try {
                text = await lastElement.innerText();
                if (attempts <= 5) {
                  console.log(`🔍 DEBUG: innerText from "${selector}": "${text ? text.substring(0, 100) : 'null'}"...`);
                }
              } catch (e) {
                if (attempts <= 5) {
                  console.log(`🔍 DEBUG: innerText failed for "${selector}":`, e.message);
                }
              }
            }
            
            if (!text || !text.trim()) {
              try {
                const innerHTML = await lastElement.innerHTML();
                // Strip HTML tags to get plain text
                text = innerHTML.replace(/<[^>]*>/g, '').replace(/&nbsp;/g, ' ').replace(/&amp;/g, '&').replace(/&lt;/g, '<').replace(/&gt;/g, '>');
                if (attempts <= 5) {
                  console.log(`🔍 DEBUG: innerHTML (stripped) from "${selector}": "${text ? text.substring(0, 100) : 'null'}"...`);
                }
              } catch (e) {
                if (attempts <= 5) {
                  console.log(`🔍 DEBUG: innerHTML failed for "${selector}":`, e.message);
                }
              }
            }
            
            if (text && text.trim() && text.trim() !== '...' && text.trim().length > 3) {
              currentContent = text.trim();
              if (attempts <= 5) {
                console.log(`🔍 DEBUG: Using content from "${selector}": ${currentContent.length} chars`);
              }
              break;
            }
          }
        } catch (e) {
          if (attempts <= 5) {
            console.log(`🔍 DEBUG: Error with selector "${selector}":`, e.message);
          }
        }
      }
      
      if (attempts <= 5 || attempts % 50 === 0) { // Reduce debug noise
        console.log(`🔍 DEBUG: Attempt ${attempts}: currentContent length = ${currentContent ? currentContent.length : 0}, isGenerating = ${isGenerating}`);
      }
      
      if (currentContent && currentContent !== lastContent) {
        const cleanedContent = cleanWebUIResponse(currentContent);
        
        // Send streaming update even for small changes
        sendEvent('content', { 
          content: cleanedContent,
          isComplete: !isGenerating
        });
        lastContent = currentContent;
        finalContent = cleanedContent;
        noChangeCount = 0;
        
        if (!isGenerating) {
          responseComplete = true;
          sendEvent('complete', { 
            content: cleanedContent,
            message: '✅ Response complete!'
          });
          console.log('✅ Response marked complete - no stop button detected');
          break;
        }
      } else if (currentContent) {
        noChangeCount++;
        
        // Reduced threshold for completion detection
        if (noChangeCount >= 5 && !isGenerating) { // Reduced from 8 to 5
          responseComplete = true;
          const cleanedContent = cleanWebUIResponse(currentContent);
          finalContent = cleanedContent;
          sendEvent('complete', { 
            content: cleanedContent,
            message: '✅ Response complete (no changes detected)!'
          });
          console.log('✅ Response marked complete - no changes for 0.5+ seconds and no stop button');
          break;
        }
      }
      
      // More frequent status updates
      if (attempts % 10 === 0) { // Reduced from 20 to 10
        res.write(`data: ${JSON.stringify({ type: 'status', message: 'still_generating', seconds: Math.floor(attempts / 10) })}\n\n`);
      }
    }
    
    if (responseComplete) {
      // Save storage state to persist login session
      try {
        await context.storageState({ path: './.puppeteer_profile/playwright_storage.json' });
        console.log('💾 Login session saved to .puppeteer_profile/playwright_storage.json');
      } catch (e) {
        console.error('❌ Error saving storage state:', e.message);
      }
      
      // Log successful response
      logResponse(headlessMode ? 'headless' : 'headful', question, finalContent, null, {
        method: 'streaming',
        attempts: attempts,
        duration: `${Math.floor(attempts/10)}s`,
        mode: headlessMode ? 'headless' : 'headful'
      });
    } else if (attempts >= maxAttempts) {
      const errorMsg = 'Response timeout - took too long to complete';
      sendEvent('error', { message: errorMsg });
      
      // Log timeout error
      logResponse(headlessMode ? 'headless' : 'headful', question, null, errorMsg, {
        method: 'streaming',
        attempts: attempts,
        timeout: true,
        mode: headlessMode ? 'headless' : 'headful'
      });
    }

  } catch (err) {
    console.error('❌ Error:', err.message);
    sendEvent('error', { message: err.message });
    
    // Log error
    logResponse(headlessMode ? 'headless' : 'headful', question, null, err.message, {
      method: 'streaming',
      errorType: 'browser_error',
      mode: headlessMode ? 'headless' : 'headful'
    });
  } finally {
    clearTimeout(operationTimeout);
    console.log('� Starting browser cleanup...');
    await forceCleanup();
    
    console.log('🏁 Cleanup complete, ending response');
    try {
      res.end();
    } catch (e) {
      console.error('❌ Error ending response:', e.message);
    }
  }
}

// Reset session endpoint
app.post('/api/reset-session', (req, res) => {
  currentSessionUrl = null;
  console.log('🗑️ Server session URL reset - next request will start new conversation');
  res.json({ success: true, message: 'Session reset successfully' });
});

// Health check endpoint
app.get('/health', (req, res) => {
  res.json({ status: 'ok', timestamp: new Date().toISOString() });
});

const PORT = process.env.PORT || 3000;
app.listen(PORT, () => {
  console.log(`✅ Simple Chat Server running on http://localhost:${PORT}`);
  console.log(`🤖 Ready to handle chat requests`);
});