ClassyGitShadowzzh

server.js

server.js

const express = require('express');
const http = require('http');
const WebSocket = require('ws');
const axios = require('axios');
const config = require('./config');
const { preprocessSentence } = require('./utils/textProcessor');
const VoiceActivityDetector = require('./utils/vad');
const SpeechToTextService = require('./utils/speechToText');
const { LLMProviderFactory } = require('./utils/providers/llmProviders');
const fs = require('fs');
const path = require('path');
const ffmpeg = require('fluent-ffmpeg');

// Import franc dynamically at the top level
let franc;
(async () => {
  const francModule = await import('franc');
  franc = francModule.franc;
})();

const app = express();
const server = http.createServer(app);
const wss = new WebSocket.Server({ server });

// Create a connection handler class to manage state for each connection
class ConnectionHandler {
  constructor(ws) {
    this.ws = ws;
    this.messageHistory = [];
    this.currentLLMRequest = null;
    this.latestImage = null;
    this.currentTTSRequest = null;
    this.lastProcessedTranscript = null;
    this.playbackStartTime = null;
    this.lastUserMessageId = null;
    this.currentMessageId = null;  // Track current message ID
    this.llmOrTTSIsWorking = false;
    this.ttsQueue = [];  // Queue for pending TTS requests
    this.isTTSProcessing = false;  // Flag to track if TTS is currently processing
    this.lastAudioEndTime = null;  // Track when the last audio chunk will finish playing
    this.audioTotalDuration = 0;   // Track total duration of queued audio

    // Add properties for audio recording
    this.audioChunks = [];
    this.isRecording = false;
    this.recordingStartTime = null;
    this.recordingPath = path.join(__dirname, 'recordings');
    
    // Create recordings directory if it doesn't exist
    if (!fs.existsSync(this.recordingPath)) {
      fs.mkdirSync(this.recordingPath, { recursive: true });
    }

    this.audioFormat = null;  // Add this property

    this.expectedPlaybackEndTime = null;  // Add this property

    this.lastSyncedHistoryLength = 0;  // Track how much of history has been synced
    this.lastSyncedHistory = [];  // Keep track of last synced state

    // Initialize VAD and STT services
    this.vad = new VoiceActivityDetector();
    this.sttService = new SpeechToTextService();
    
    // Initialize LLM provider
    this.initializeLLMProvider();
    
    // VAD processing state
    this.isProcessingSTT = false;
    this.pendingAudioBuffer = Buffer.alloc(0);

    this.setupWebSocketHandlers();
    
    // Start periodic cleanup of temp files
    this.cleanupInterval = setInterval(() => {
      this.sttService.cleanupTempFiles();
    }, 5 * 60 * 1000); // Every 5 minutes
  }

  /**
   * Initialize LLM provider based on configuration
   */
  initializeLLMProvider() {
    try {
      const providerName = config.LLM_PROVIDER || 'openai';
      this.llmProvider = LLMProviderFactory.createProvider(providerName, config, config);
      console.log(`LLM Provider initialized: ${providerName}`);
    } catch (error) {
      console.error('Failed to initialize LLM provider:', error);
      throw error;
    }
  }

  /**
   * Switch LLM provider dynamically
   * @param {string} providerName - Provider name to switch to
   */
  switchLLMProvider(providerName) {
    try {
      this.llmProvider = LLMProviderFactory.createProvider(providerName, config, config);
      console.log(`LLM Provider switched to: ${providerName}`);
    } catch (error) {
      console.error('Failed to switch LLM provider:', error);
      throw error;
    }
  }

  setupWebSocketHandlers() {
    this.ws.on('message', this.handleMessage.bind(this));
    this.ws.on('close', this.handleClose.bind(this));
    this.ws.on('error', this.handleError.bind(this));
  }

  async handleMessage(message) {
    try {
      // Try to parse as JSON first
      const jsonMessage = JSON.parse(message);
      
      // Handle ping message by sending back pong with same timestamp
      if (jsonMessage.type === 'ping') {
        this.ws.send(JSON.stringify({
          type: 'pong',
          timestamp: jsonMessage.timestamp
        }));
        return;
      }

      // Handle image message
      if (jsonMessage.type === 'image') {
        this.latestImage = jsonMessage.data;
        console.log('Received new image from camera');
        return;
      }

      // Handle other ping message by sending back pong with same timestamp
      if (jsonMessage.type === 'ping') {
        this.ws.send(JSON.stringify({
          type: 'pong',
          timestamp: jsonMessage.timestamp
        }));
        return;
      }

      console.log('Received JSON message:', jsonMessage);
    } catch (e) {
      // If not JSON, treat as binary data
      if (Buffer.isBuffer(message)) {
        // Start recording if not already started
        if (!this.isRecording) {
          this.startRecording();
        }
        
        // Store the audio chunk
        this.audioChunks.push(message);

        // Process audio through VAD
        this.processAudioWithVAD(message);
      } else {
        console.error('Received unexpected data type:', typeof message);
      }
    }
  }

  handleClose() {
    console.log('Client disconnected');
    this.cleanup();
  }

  handleError(error) {
    console.error('WebSocket error:', error);
    this.cleanup();
  }

  cleanup() {
    if (this.currentLLMRequest) {
      this.currentLLMRequest.cancel();
    }
    if (this.currentTTSRequest) {
      this.currentTTSRequest.cancel();
    }
    this.ttsQueue = [];
    this.lastAudioEndTime = null;
    this.audioTotalDuration = 0;

    // Clean up VAD state (now async)
    if (this.vad) {
      this.vad.cleanup().catch(error => {
        console.error('Error cleaning up VAD:', error);
      });
    }

    // Clear cleanup interval
    if (this.cleanupInterval) {
      clearInterval(this.cleanupInterval);
    }

    // Save any remaining recording
    if (this.isRecording) {
      this.isRecording = false;
      this.saveRecording();
      this.audioChunks = [];
    }

    // Clear pending audio buffer
    this.pendingAudioBuffer = Buffer.alloc(0);
  }

  logEvent(type, details = {}) {
    const timestamp = new Date().toISOString();
    console.log(`[${timestamp}] Event: ${type}`, {
      ...details,
      connectionId: this.ws._socket.remoteAddress
    });
  }

  // Add helper method to generate message IDs
  generateMessageId() {
    return Math.random().toString(36).substring(2, 15);
  }

  /**
   * Process audio chunk through VAD
   * @param {Buffer} audioChunk - Raw audio data
   */
  async processAudioWithVAD(audioChunk) {
    try {
      // Process audio through VAD (now async)
      const vadResults = await this.vad.processAudioChunk(audioChunk);
      
      for (const result of vadResults) {
        if (result.type === 'speech_end') {
          // Speech segment ended, process with STT
          await this.processSpeechSegment(result.audioData, result.duration);
        }
      }
    } catch (error) {
      console.error('Error processing audio with VAD:', error);
    }
  }

  /**
   * Process speech segment with STT
   * @param {Buffer} audioData - Speech audio data
   * @param {number} duration - Duration of speech in ms
   */
  async processSpeechSegment(audioData, duration) {
    if (this.isProcessingSTT) {
      console.log('STT already processing, skipping this segment');
      return;
    }

    try {
      this.isProcessingSTT = true;
      
      // Check if audio has sufficient content
      if (!this.sttService.hasSufficientAudio(audioData)) {
        console.log('Insufficient audio content for STT');
        return;
      }

      console.log(`Processing speech segment: ${duration}ms, ${audioData.length} bytes`);
      
      // Send processing start notification
      this.ws.send(JSON.stringify({
        type: 'stt_start',
        duration: duration,
        timestamp: Date.now()
      }));

      // Generate message ID for this speech segment
      this.currentMessageId = this.generateMessageId();
      
      // Transcribe audio
      const result = await this.sttService.transcribeAudio(audioData);
      
      if (result.success && result.text.trim()) {
        const transcript = result.text.trim();
        
        // Send transcript result
        this.ws.send(JSON.stringify({
          type: 'transcript',
          text: transcript,
          isFinal: true,
          messageId: this.currentMessageId,
          language: result.language,
          duration: result.duration,
          confidence: result.confidence
        }));
        
        this.logEvent('transcript', { 
          text: transcript, 
          isFinal: true, 
          messageId: this.currentMessageId,
          language: result.language,
          sttDuration: result.duration
        });

        // Only process if this transcript is different from the last one we processed
        if (transcript !== this.lastProcessedTranscript) {
          this.lastProcessedTranscript = transcript;
          
          // Update message history
          const lastMessage = this.messageHistory[this.messageHistory.length - 1];
          const isLastMessageUser = lastMessage && 
            (lastMessage.role === 'user' || lastMessage.role === 'transcript');

          if (isLastMessageUser) {
            // Replace the last user message
            this.messageHistory = [
              ...this.messageHistory.slice(0, -1),
              { role: 'user', content: transcript, messageId: this.currentMessageId }
            ];
          } else {
            // Append new message
            this.messageHistory.push({ 
              role: 'user', 
              content: transcript,
              messageId: this.currentMessageId 
            });
          }

          this.syncChatHistory();
          this.generateAIResponse();
        }
      } else if (!result.success) {
        console.error('STT failed:', result.error);
        this.ws.send(JSON.stringify({
          type: 'stt_error',
          error: result.error,
          timestamp: Date.now()
        }));
      }
      
    } catch (error) {
      console.error('Error processing speech segment:', error);
      this.ws.send(JSON.stringify({
        type: 'stt_error',
        error: error.message,
        timestamp: Date.now()
      }));
    } finally {
      this.isProcessingSTT = false;
    }
  }

  async generateAIResponse() {
    this.llmOrTTSIsWorking = true;
    try {
      this.currentLLMRequest = axios.CancelToken.source();
      
      this.ws.send(JSON.stringify({ type: 'llm_start' }));
      this.logEvent('llm_start');

      let hasReceivedFirstToken = false;
      let hasReceivedFirstSentence = false;
      let currentSentence = '';
      let isFirstSentence = true;
      
      // Keep only the last 20 messages for context
      const recentHistory = this.messageHistory.slice(-20);
      let messages = [
        { role: 'system', content: config.SYSTEM_PROMPT },
        ...recentHistory.map(({ role, content }) => ({ role, content }))
      ];

      // If we have a latest image, include it in the messages
      if (this.latestImage) {
        // Find the last user message
        const lastUserMessageIndex = messages.findIndex(msg => msg.role === 'user');
        if (lastUserMessageIndex !== -1) {
          // Add image to the user's message
          messages[lastUserMessageIndex] = {
            role: 'user',
            content: [
              {
                type: 'image_url',
                image_url: {
                  url: this.latestImage,
                  detail: 'low'
                }
              },
              {
                type: 'text',
                text: messages[lastUserMessageIndex].content
              }
            ]
          };
        }
        // Clear the image after using it
        this.latestImage = null;
      }

      // Add initial empty assistant message
      this.messageHistory.push({
        role: 'assistant',
        content: '',
        messageId: this.currentMessageId
      });
      this.syncChatHistory();

      let accumulatedContent = '';  // Track all content received so far

      // Use LLM provider for chat completion
      const providerResult = await this.llmProvider.createChatCompletion(messages, {
        max_tokens: config.VISION_MAX_TOKENS,
        cancelToken: this.currentLLMRequest.token
      });

      if (!providerResult.success) {
        throw new Error(providerResult.error);
      }

      const response = providerResult.response;

      let buffer = ''; // Add this buffer to store incomplete lines

      response.data.on('data', async chunk => {
        try {
          buffer += chunk.toString();
          const lines = buffer.split('\n');
          // Keep the last line if it's incomplete
          buffer = lines.pop() || '';

          for (const line of lines) {
            const trimmedLine = line.trim();
            if (!trimmedLine || trimmedLine === '[DONE]') continue;
            if (!trimmedLine.startsWith('data: ')) continue;

            try {
              const jsonData = JSON.parse(trimmedLine.replace('data: ', ''));
              const content = jsonData.choices[0]?.delta?.content || '';
              if (!content) continue;

              if (!hasReceivedFirstToken) {
                hasReceivedFirstToken = true;
                this.ws.send(JSON.stringify({ 
                  type: 'llm_first_token',
                  messageId: this.currentMessageId
                }));
              }

              currentSentence += content;
              accumulatedContent += content;

              if (this.isCompleteSentence(currentSentence, isFirstSentence)) {
                // Update message history with accumulated content after each complete sentence
                const lastMessage = this.messageHistory[this.messageHistory.length - 1];
                lastMessage.content = accumulatedContent;
                this.syncChatHistory();

                const completedSentence = currentSentence;
                currentSentence = '';
                isFirstSentence = false;

                this.ws.send(JSON.stringify({ 
                  type: 'llm_sentence', 
                  text: completedSentence,
                  messageId: this.currentMessageId
                }));

                if (!hasReceivedFirstSentence) {
                  hasReceivedFirstSentence = true;
                  this.ws.send(JSON.stringify({ 
                    type: 'llm_first_sentence',
                    messageId: this.currentMessageId
                  }));
                }

                await this.synthesizeAndStreamAudio(completedSentence);
              }
            } catch (parseError) {
              console.error('Error parsing JSON data:', parseError);
              continue;
            }
          }
        } catch (error) {
          console.error('Error processing LLM stream:', error);
        }
      });

      // Handle any remaining data in the buffer when the stream ends
      response.data.on('end', async () => {
        try {
          if (buffer) {
            const trimmedLine = buffer.trim();
            if (trimmedLine && trimmedLine !== '[DONE]' && trimmedLine.startsWith('data: ')) {
              const jsonData = JSON.parse(trimmedLine.replace('data: ', ''));
              const content = jsonData.choices[0]?.delta?.content || '';
              if (content) {
                currentSentence += content;
                accumulatedContent += content;
              }
            }
          }

          // Handle any remaining content
          if (currentSentence.trim()) {
            await this.synthesizeAndStreamAudio(currentSentence);
            accumulatedContent += currentSentence;
            
            // Update message history with final content
            const lastMessage = this.messageHistory[this.messageHistory.length - 1];
            lastMessage.content = accumulatedContent;
            this.syncChatHistory();
          }

          this.ws.send(JSON.stringify({ 
            type: 'ai_response_complete',
            messageId: this.currentMessageId
          }));
        } catch (error) {
          console.error('Error processing final LLM data:', error);
        }
      });

    } catch (error) {
      this.llmOrTTSIsWorking = false;
      if (!axios.isCancel(error)) {
        console.error('Error generating AI response:', error);
        this.ws.send(JSON.stringify({ type: 'error', message: 'Error generating AI response' }));
        this.logEvent('error', { message: 'Error generating AI response', error: error.toString() });
      }
    }
  }

  isCompleteSentence(sentence, isFirstSentence) {
    const trimmedSentence = sentence.trim();

    // Check if we're inside a markdown code block
    if (trimmedSentence.includes('```')) {
      const count = (trimmedSentence.match(/```/g) || []).length;
      // If the count is odd, we're still inside a code block
      if (count % 2 !== 0) {
        return false;
      }
    }

    // Check if the sentence is a function call
    if (trimmedSentence.includes('<function>')) {
      return trimmedSentence.includes('</function>');
    }

    // Check for newline
    if (sentence.endsWith('\n')) {
      return true;
    }

    // Check for period, but not if it's a numbered list item
    if (trimmedSentence.endsWith('.')) {
      if (/[0-9]+\.$/.test(trimmedSentence)) {
        return false;
      }
      return true;
    }

    // Check for question mark or exclamation mark
    if (trimmedSentence.endsWith('?') || trimmedSentence.endsWith('!')) {
      return true;
    }

    // Check for Chinese punctuation
    if (trimmedSentence.endsWith('。') || trimmedSentence.endsWith('?') || trimmedSentence.endsWith('!')) {
      return true;
    }

    // Check for semicolons (both English and Chinese)
    if (trimmedSentence.endsWith(';') || trimmedSentence.endsWith(';')) {
      return true;
    }

    // Check if sentence ends with an emoji
    if (trimmedSentence.length > 0) {
      const lastChar = trimmedSentence.slice(-2); // Take last 2 chars for emoji
      const emojiRegex = /[\u{1F300}-\u{1F9FF}]|[\u{2600}-\u{26FF}]/u;
      if (emojiRegex.test(lastChar)) {
        return true;
      }
    }

    // First sentence should be as short as possible
    if (isFirstSentence) {
      if (trimmedSentence.endsWith(',') || trimmedSentence.endsWith(',')) {
        return true;
      }
    }

    return false;
  }

  async synthesizeAndStreamAudio(text) {
    try {
      // Skip empty or whitespace-only strings
      if (!text || !text.trim()) {
        console.log('Skipping TTS for empty string');
        return;
      }

      // Add text to the queue
      this.ttsQueue.push(text);
      
      // Try to process the queue
      await this.processTTSQueue();
    } catch (error) {
      if (!axios.isCancel(error)) {
        console.error('Error in TTS:', error);
        this.logEvent('error', { message: 'TTS error', error: error.toString() });
      }
      this.isTTSProcessing = false;
    }
  }

  // Helper method to calculate audio duration from WAV buffer
  calculateAudioDuration(audioBuffer) {
    if (audioBuffer.length < 44) return 0;
    
    const sampleRate = audioBuffer.readUInt32LE(24);
    const numChannels = audioBuffer.readUInt16LE(22);
    const bitsPerSample = audioBuffer.readUInt16LE(34);
    const dataSize = audioBuffer.length - 44; // Subtract WAV header size
    
    // Calculate duration in milliseconds
    const duration = Math.floor(
      (dataSize * 8 * 1000) / (sampleRate * numChannels * bitsPerSample)
    );
    
    return duration;
  }

  // Add method to start a new recording
  startRecording() {
    this.audioChunks = [];
    this.isRecording = true;
    this.recordingStartTime = new Date();
  }

  // Add method to save the recording
  saveRecording() {
    if (this.audioChunks.length === 0) return;

    const timestamp = this.recordingStartTime.toISOString().replace(/[:.]/g, '-');
    const fileName = `recording_${timestamp}.wav`;
    const filePath = path.join(this.recordingPath, fileName);

    // Create WAV header
    const dataSize = this.audioChunks.reduce((acc, chunk) => acc + chunk.length, 0);
    const header = Buffer.alloc(44);

    // RIFF chunk descriptor
    header.write('RIFF', 0);
    header.writeUInt32LE(36 + dataSize, 4);
    header.write('WAVE', 8);

    // fmt sub-chunk
    header.write('fmt ', 12);
    header.writeUInt32LE(16, 16); // Subchunk1Size
    header.writeUInt16LE(1, 20); // AudioFormat (PCM)
    header.writeUInt16LE(1, 22); // NumChannels (Mono)
    header.writeUInt32LE(16000, 24); // SampleRate
    header.writeUInt32LE(32000, 28); // ByteRate (SampleRate * NumChannels * BitsPerSample/8)
    header.writeUInt16LE(2, 32); // BlockAlign (NumChannels * BitsPerSample/8)
    header.writeUInt16LE(16, 34); // BitsPerSample

    // data sub-chunk
    header.write('data', 36);
    header.writeUInt32LE(dataSize, 40);

    // Write header and audio data to file
    const writeStream = fs.createWriteStream(filePath);
    writeStream.write(header);

    for (const chunk of this.audioChunks) {
      writeStream.write(chunk);
    }

    writeStream.end();
    this.logEvent('recording_saved', { filePath, size: dataSize + 44 });
  }

  async detectLanguage(text) {
    try {
      if (!franc) {
        const francModule = await import('franc');
        franc = francModule.franc;
      }

      const detectedLang = franc(text, { minLength: 1 });
      const langMap = {
        'cmn': 'zh',
        'eng': 'en',
        'jpn': 'jp',
        'zho': 'zh',
        'chi': 'zh',
        'und': 'en'
      };
      
      return langMap[detectedLang] || 'en';
    } catch (error) {
      console.error('Language detection error:', error);
      return 'en';
    }
  }



  // Add this method to process TTS queue
  async processTTSQueue() {
    if (this.isTTSProcessing || this.ttsQueue.length === 0) return;

    this.isTTSProcessing = true;
    const now = Date.now();

    // Calculate remaining audio duration
    const remainingDuration = this.lastAudioEndTime 
      ? Math.max(0, this.lastAudioEndTime - now) 
      : 0;

    // Only process next TTS if remaining audio is less than 5 seconds
    if (remainingDuration <= 5000) {
      const text = this.ttsQueue[0];
      const ttsStartTime = Date.now();
      
      try {
        this.ws.send(JSON.stringify({ type: 'tts_start' }));
        this.logEvent('tts_start');
        
        const detectedLang = await this.detectLanguage(text);
        this.logEvent('language_detection', { text, detectedLang });
        
        const processedText = preprocessSentence(text, detectedLang);
        // Skip TTS if processed text is empty
        if (!processedText.trim()) {
          this.ttsQueue.shift(); // Remove empty text from queue
          this.isTTSProcessing = false;
          // Process next item in queue if any
          if (this.ttsQueue.length > 0) {
            setTimeout(() => this.processTTSQueue(), 100);
          }
          return;
        }

        this.currentTTSRequest = axios.CancelToken.source();
        const response = await axios({
          method: 'post',
          url: config.TTS_API_URL,
          data: {
            "model": (config.TTS_PROVIDERS?.[config.TTS_PROVIDER]?.model) || "FunAudioLLM/CosyVoice2-0.5B",
            "input": text,
            "voice": (config.TTS_PROVIDERS?.[config.TTS_PROVIDER]?.voice) || "FunAudioLLM/CosyVoice2-0.5B:diana",
            "response_format": "mp3",
            "sample_rate": 32000,
            "stream": true,
            "speed": 1,
            "gain": 0
          },
          headers: {
            "Authorization": `Bearer ${config.SILICONFLOW_API_KEY}`,
            "Content-Type": "application/json"
          },
          responseType: 'stream',
          cancelToken: this.currentTTSRequest.token,
        });

        let headerBuffer = Buffer.alloc(0);
        let headerSent = false;
        let totalDataSize = 0;

        response.data.on('data', async chunk => {
          if (!headerSent) {
            headerBuffer = Buffer.concat([headerBuffer, chunk]);
            
            if (headerBuffer.length >= 44) {
              const sampleRate = headerBuffer.readUInt32LE(24);
              const numChannels = headerBuffer.readUInt16LE(22);
              const bitsPerSample = headerBuffer.readUInt16LE(34);

              this.audioFormat = { sampleRate, numChannels, bitsPerSample };

              const needsResampling = sampleRate !== 16000 || numChannels !== 1;

              if (needsResampling) {
                this.audioBuffer = Buffer.from(headerBuffer);
                headerBuffer = Buffer.alloc(0);
                headerSent = true;
                return;
              }

              const bytesPerSample = bitsPerSample / 8;
              const samplesPerSecond = sampleRate * numChannels;
              const bytesPerSecond = samplesPerSecond * bytesPerSample;
              const chunkSize = Math.floor(bytesPerSecond * 0.05); // 50ms worth of audio data

              this.ws.send(JSON.stringify({ 
                type: 'audio_start',
                format: {
                  sampleRate,
                  numChannels,
                  bitsPerSample
                }
              }));

              // Set playback start time immediately
              if (!this.playbackStartTime) {
                this.playbackStartTime = Date.now();
              }
              // Start sending audio data immediately (skip the header and data chunk header)
              if (headerBuffer.length > 44) {
                let remainingData = headerBuffer.slice(44);
                
                // Skip data chunk header (8 bytes) if present
                if (remainingData.length >= 8 && 
                    remainingData.slice(0, 4).toString() === 'data') {
                  remainingData = remainingData.slice(8);
                }

                while (remainingData.length >= chunkSize) {
                  const chunk = remainingData.slice(0, chunkSize);
                  this.ws.send(chunk);
                  totalDataSize += chunk.length;
                  remainingData = remainingData.slice(chunkSize);
                }
                headerBuffer = remainingData.length > 0 ? remainingData : Buffer.alloc(0);
              }

              headerSent = true;
            }
          } else {
            if (this.audioFormat.sampleRate !== 16000 || this.audioFormat.numChannels !== 1) {
              this.audioBuffer = Buffer.concat([this.audioBuffer, chunk]);
            } else {
              const bytesPerSample = this.audioFormat.bitsPerSample / 8;
              const samplesPerSecond = this.audioFormat.sampleRate * this.audioFormat.numChannels;
              const bytesPerSecond = samplesPerSecond * bytesPerSample;
              const chunkSize = Math.floor(bytesPerSecond * 0.05);

              let audioData = Buffer.concat([headerBuffer, chunk]);
              headerBuffer = Buffer.alloc(0);

              while (audioData.length >= chunkSize) {
                const chunkToSend = audioData.slice(0, chunkSize);
                this.ws.send(chunkToSend);
                totalDataSize += chunkSize;
                audioData = audioData.slice(chunkSize);
              }

              if (audioData.length > 0) {
                headerBuffer = audioData;
              }
            }
          }
        });

        response.data.on('end', async () => {
          try {
            if (this.audioFormat.sampleRate !== 16000 || this.audioFormat.numChannels !== 1) {
              // Create temporary files for resampling
              const inputPath = path.join(__dirname, `temp_input_${Date.now()}.wav`);
              const outputPath = path.join(__dirname, `temp_output_${Date.now()}.wav`);

              // Write the input audio buffer to a temporary file
              fs.writeFileSync(inputPath, this.audioBuffer);

              // Create a promise to handle the ffmpeg conversion
              await new Promise((resolve, reject) => {
                ffmpeg(inputPath)
                  .toFormat('wav')
                  .outputOptions([
                    '-acodec pcm_s16le',  // Set codec to 16-bit PCM
                    '-ar 16000',          // Set sample rate to 16kHz
                    '-ac 1'               // Set to mono channel
                  ])
                  .on('error', (err) => {
                    console.error('FFmpeg error:', err);
                    reject(err);
                  })
                  .on('end', () => resolve())
                  .save(outputPath);
              });

              // Read the resampled audio
              const resampledBuffer = fs.readFileSync(outputPath);

              // Clean up temporary files
              fs.unlinkSync(inputPath);
              fs.unlinkSync(outputPath);

              // Send audio start with resampled format
              this.ws.send(JSON.stringify({
                type: 'audio_start',
                format: {
                  sampleRate: 16000,
                  numChannels: 1,
                  bitsPerSample: 16
                }
              }));

              // Set playback start time
              if (!this.playbackStartTime) {
                this.playbackStartTime = Date.now();
              }

              // Find the data chunk in the resampled buffer
              let dataStart = 44; // Start looking after standard WAV header
              while (dataStart < resampledBuffer.length - 8) {
                if (resampledBuffer.readUInt32LE(dataStart) === 0x61746164) { // "data" chunk ID
                  dataStart += 8; // Skip "data" ID (4 bytes) and chunk size (4 bytes)
                  break;
                }
                dataStart++;
              }

              // Calculate chunk size for 50ms of audio at 16kHz mono
              const chunkSize = Math.floor(16000 * 2 * 0.05); // 16kHz * 16-bit * 50ms

              // Send audio data in chunks, starting after the data header
              let currentPos = dataStart;
              while (currentPos < resampledBuffer.length) {
                const end = Math.min(currentPos + chunkSize, resampledBuffer.length);
                const audioChunk = resampledBuffer.slice(currentPos, end);
                this.ws.send(audioChunk);
                totalDataSize += audioChunk.length;
                currentPos += chunkSize;
              }

            } else {
              // For non-resampled audio, process the remaining audio buffer
              const audioData = this.audioBuffer;
              if (audioData) {
                // Calculate chunk size for 50ms of audio
                const chunkSize = Math.floor(16000 * 2 * 0.05);

                // Send audio data in chunks
                let currentPos = dataStart;
                while (currentPos < audioData.length) {
                  const end = Math.min(currentPos + chunkSize, audioData.length);
                  const audioChunk = audioData.slice(currentPos, end);
                  this.ws.send(audioChunk);
                  totalDataSize += audioChunk.length;
                  currentPos += chunkSize;
                }
              }
            }

            const synthesisTime = Date.now() - ttsStartTime;
            this.ws.send(JSON.stringify({ 
              type: 'tts_complete',
              synthesisTime 
            }));
            this.ws.send(JSON.stringify({ type: 'audio_end' }));
            this.logEvent('tts_complete', { synthesisTime });

            // Calculate audio duration and update expected playback end time
            const audioDuration = Math.floor((totalDataSize * 8 * 1000) / (16000 * 1 * 16));
            this.expectedPlaybackEndTime = (this.expectedPlaybackEndTime || Date.now()) + audioDuration;
            this.lastAudioEndTime = Date.now() + audioDuration;
            
            // Remove the processed text from queue
            this.ttsQueue.shift();
            this.isTTSProcessing = false;
            
            // Process next item in queue if any
            if (this.ttsQueue.length > 0) {
              setTimeout(() => this.processTTSQueue(), 100);
            }
          } catch (error) {
            console.error('Error processing audio:', error);
            this.logEvent('error', { message: 'Audio processing error', error: error.toString() });
            
            // Clean up state
            this.ttsQueue.shift();
            this.isTTSProcessing = false;
          }
        });

        response.data.on('error', (error) => {
          console.error('Error in TTS stream:', error);
          this.logEvent('error', { message: 'TTS stream error', error: error.toString() });
          this.ttsQueue.shift();  // Remove failed text from queue
          this.isTTSProcessing = false;
        });

      } catch (error) {
        if (!axios.isCancel(error)) {
          console.error('Error in TTS:', error);
          this.logEvent('error', { message: 'TTS error', error: error.toString() });
        }
        this.ttsQueue.shift();  // Remove failed text from queue
        this.isTTSProcessing = false;
      }
    } else {
      this.isTTSProcessing = false;
      // Try again later if there's still too much audio in the queue
      setTimeout(() => this.processTTSQueue(), 100);
    }
  }

  // Update the syncChatHistory method to implement delta sync
  syncChatHistory() {
    const currentHistory = this.messageHistory;
    const lastHistory = this.lastSyncedHistory;
    
    // Find the index where histories start to differ
    let diffStartIndex = 0;
    while (diffStartIndex < lastHistory.length && 
           diffStartIndex < currentHistory.length && 
           JSON.stringify(lastHistory[diffStartIndex]) === JSON.stringify(currentHistory[diffStartIndex])) {
      diffStartIndex++;
    }

    // Get the new or modified messages
    const updatedMessages = currentHistory.slice(diffStartIndex);
    
    // Send delta update
    this.ws.send(JSON.stringify({
      type: 'chat_history_delta',
      startIndex: diffStartIndex,
      messages: updatedMessages
    }));

    // Update last synced state
    this.lastSyncedHistory = JSON.parse(JSON.stringify(currentHistory));
  }
}

// Handle new WebSocket connections
wss.on('connection', (ws) => {
  console.log('Client connected');
  new ConnectionHandler(ws);
});

server.listen(config.LISTEN_PORT, config.LISTEN_HOST, () => {
  console.log(`Server is running on ${config.LISTEN_HOST}:${config.LISTEN_PORT}`);
});