WebRTC Media Streams

Media Capture APIs

WebRTC provides powerful APIs for capturing and streaming audio and video. The getUserMedia API captures camera/microphone, while getDisplayMedia captures screen content. These streams can then be sent peer-to-peer via WebRTC.

getUserMedia - Camera & Microphone

// Request access to camera and microphone

// Basic usage
async function startCamera() {
  try {
    const stream = await navigator.mediaDevices.getUserMedia({
      video: true,
      audio: true
    });
    
    // Attach to video element
    document.getElementById('localVideo').srcObject = stream;
    
    return stream;
  } catch (error) {
    handleMediaError(error);
  }
}

function handleMediaError(error) {
  switch (error.name) {
    case 'NotAllowedError':
      alert('Permission denied. Please allow camera/microphone access.');
      break;
    case 'NotFoundError':
      alert('No camera or microphone found.');
      break;
    case 'NotReadableError':
      alert('Camera/microphone is already in use.');
      break;
    case 'OverconstrainedError':
      alert('Requested constraints cannot be satisfied.');
      break;
    default:
      console.error('Media error:', error);
  }
}

// With detailed constraints
const stream = await navigator.mediaDevices.getUserMedia({
  video: {
    width: { min: 640, ideal: 1280, max: 1920 },
    height: { min: 480, ideal: 720, max: 1080 },
    frameRate: { ideal: 30, max: 60 },
    facingMode: 'user',  // 'user' (front) or 'environment' (back)
    aspectRatio: 16/9
  },
  audio: {
    echoCancellation: true,
    noiseSuppression: true,
    autoGainControl: true,
    sampleRate: 48000,
    channelCount: 2
  }
});

// Enumerate available devices
async function getDevices() {
  const devices = await navigator.mediaDevices.enumerateDevices();
  
  const cameras = devices.filter(d => d.kind === 'videoinput');
  const microphones = devices.filter(d => d.kind === 'audioinput');
  const speakers = devices.filter(d => d.kind === 'audiooutput');
  
  console.log('Cameras:', cameras);
  console.log('Microphones:', microphones);
  console.log('Speakers:', speakers);
  
  return { cameras, microphones, speakers };
}

// Select specific device
async function useCamera(deviceId) {
  return navigator.mediaDevices.getUserMedia({
    video: { deviceId: { exact: deviceId } },
    audio: true
  });
}

getDisplayMedia - Screen Sharing

// Capture screen, window, or browser tab

async function startScreenShare() {
  try {
    const stream = await navigator.mediaDevices.getDisplayMedia({
      video: {
        cursor: 'always',  // 'always', 'motion', or 'never'
        displaySurface: 'monitor'  // 'monitor', 'window', 'browser'
      },
      audio: true  // System audio (if supported)
    });
    
    // User selected a source - attach to video
    document.getElementById('screenVideo').srcObject = stream;
    
    // Handle user stopping share via browser UI
    stream.getVideoTracks()[0].onended = () => {
      console.log('User stopped screen sharing');
      handleScreenShareEnd();
    };
    
    return stream;
  } catch (error) {
    if (error.name === 'NotAllowedError') {
      console.log('User cancelled screen share');
    }
  }
}

// Combine camera and screen (picture-in-picture effect)
async function createPipStream() {
  const cameraStream = await navigator.mediaDevices.getUserMedia({
    video: { width: 320, height: 180 },
    audio: true
  });
  
  const screenStream = await navigator.mediaDevices.getDisplayMedia({
    video: true
  });
  
  // Use Canvas to combine streams
  const canvas = document.createElement('canvas');
  canvas.width = 1920;
  canvas.height = 1080;
  const ctx = canvas.getContext('2d');
  
  const screenVideo = document.createElement('video');
  screenVideo.srcObject = screenStream;
  await screenVideo.play();
  
  const cameraVideo = document.createElement('video');
  cameraVideo.srcObject = cameraStream;
  await cameraVideo.play();
  
  function draw() {
    // Draw screen full size
    ctx.drawImage(screenVideo, 0, 0, canvas.width, canvas.height);
    // Draw camera in corner
    ctx.drawImage(cameraVideo, canvas.width - 340, canvas.height - 200, 320, 180);
    requestAnimationFrame(draw);
  }
  draw();
  
  // Capture canvas as stream
  const combinedStream = canvas.captureStream(30);
  // Add audio from camera
  combinedStream.addTrack(cameraStream.getAudioTracks()[0]);
  
  return combinedStream;
}

Working with MediaStream

// MediaStream contains MediaStreamTracks

const stream = await navigator.mediaDevices.getUserMedia({ video: true, audio: true });

// Get tracks
const videoTracks = stream.getVideoTracks();
const audioTracks = stream.getAudioTracks();
const allTracks = stream.getTracks();

console.log('Video tracks:', videoTracks.length);
console.log('Audio tracks:', audioTracks.length);

// Track properties
const videoTrack = videoTracks[0];
console.log('Track ID:', videoTrack.id);
console.log('Track kind:', videoTrack.kind);
console.log('Track label:', videoTrack.label);
console.log('Track enabled:', videoTrack.enabled);
console.log('Track muted:', videoTrack.muted);
console.log('Track readyState:', videoTrack.readyState);

// Get current settings
const settings = videoTrack.getSettings();
console.log('Resolution:', settings.width, 'x', settings.height);
console.log('Frame rate:', settings.frameRate);
console.log('Device ID:', settings.deviceId);

// Get capabilities
const capabilities = videoTrack.getCapabilities();
console.log('Max resolution:', capabilities.width.max, 'x', capabilities.height.max);

// Mute/unmute (disable track)
function toggleVideo() {
  videoTrack.enabled = !videoTrack.enabled;
}

function toggleAudio() {
  audioTracks[0].enabled = !audioTracks[0].enabled;
}

// Stop tracks (release camera/mic)
function stopMedia() {
  stream.getTracks().forEach(track => {
    track.stop();
  });
}

// Track events
videoTrack.onended = () => {
  console.log('Track ended (device disconnected or stopped)');
};

videoTrack.onmute = () => {
  console.log('Track muted');
};

videoTrack.onunmute = () => {
  console.log('Track unmuted');
};

Adding Media to Peer Connection

// Send media tracks over WebRTC

const pc = new RTCPeerConnection(config);
const stream = await navigator.mediaDevices.getUserMedia({ video: true, audio: true });

// Add tracks to peer connection
stream.getTracks().forEach(track => {
  pc.addTrack(track, stream);
  console.log('Added track:', track.kind);
});

// Alternative: addStream (deprecated but still works)
// pc.addStream(stream);

// Handle incoming tracks from remote peer
pc.ontrack = (event) => {
  console.log('Received track:', event.track.kind);
  
  // event.streams contains the MediaStream(s)
  const remoteStream = event.streams[0];
  document.getElementById('remoteVideo').srcObject = remoteStream;
  
  // Or create new stream from track
  if (!remoteVideo.srcObject) {
    remoteVideo.srcObject = new MediaStream();
  }
  remoteVideo.srcObject.addTrack(event.track);
};

// Get senders (for modifying tracks later)
const senders = pc.getSenders();
const videoSender = senders.find(s => s.track?.kind === 'video');
const audioSender = senders.find(s => s.track?.kind === 'audio');

// Replace track (e.g., switch camera)
async function switchCamera(newDeviceId) {
  const newStream = await navigator.mediaDevices.getUserMedia({
    video: { deviceId: { exact: newDeviceId } }
  });
  const newTrack = newStream.getVideoTracks()[0];
  
  await videoSender.replaceTrack(newTrack);
  
  // Update local preview
  localVideo.srcObject = newStream;
}

// Add screen share while keeping audio
async function addScreenShare() {
  const screenStream = await navigator.mediaDevices.getDisplayMedia({ video: true });
  const screenTrack = screenStream.getVideoTracks()[0];
  
  // Replace video track with screen
  await videoSender.replaceTrack(screenTrack);
  
  // When screen share ends, switch back to camera
  screenTrack.onended = async () => {
    const cameraStream = await navigator.mediaDevices.getUserMedia({ video: true });
    await videoSender.replaceTrack(cameraStream.getVideoTracks()[0]);
  };
}

Complete Video Call Implementation

// Full video call with all features

class VideoCall {
  constructor(signalingSocket) {
    this.socket = signalingSocket;
    this.pc = null;
    this.localStream = null;
    this.remoteStream = null;
    this.isMuted = false;
    this.isCameraOff = false;
    
    this.config = {
      iceServers: [
        { urls: 'stun:stun.l.google.com:19302' },
        { 
          urls: 'turn:turn.example.com:3478',
          username: 'user',
          credential: 'pass'
        }
      ]
    };
    
    this.setupSignaling();
  }
  
  setupSignaling() {
    this.socket.on('offer', async ({ from, offer }) => {
      await this.handleOffer(from, offer);
    });
    
    this.socket.on('answer', async ({ answer }) => {
      await this.pc.setRemoteDescription(new RTCSessionDescription(answer));
    });
    
    this.socket.on('ice-candidate', async ({ candidate }) => {
      if (candidate) {
        await this.pc.addIceCandidate(new RTCIceCandidate(candidate));
      }
    });
    
    this.socket.on('hang-up', () => {
      this.endCall();
    });
  }
  
  async startLocalMedia() {
    this.localStream = await navigator.mediaDevices.getUserMedia({
      video: { width: 1280, height: 720 },
      audio: {
        echoCancellation: true,
        noiseSuppression: true
      }
    });
    
    document.getElementById('localVideo').srcObject = this.localStream;
    return this.localStream;
  }
  
  createPeerConnection() {
    this.pc = new RTCPeerConnection(this.config);
    
    // Add local tracks
    this.localStream.getTracks().forEach(track => {
      this.pc.addTrack(track, this.localStream);
    });
    
    // ICE candidates
    this.pc.onicecandidate = (event) => {
      if (event.candidate) {
        this.socket.emit('ice-candidate', {
          to: this.remotePeer,
          candidate: event.candidate
        });
      }
    };
    
    // Connection state
    this.pc.onconnectionstatechange = () => {
      const state = this.pc.connectionState;
      this.onConnectionStateChange?.(state);
      
      if (state === 'failed' || state === 'closed') {
        this.endCall();
      }
    };
    
    // Remote stream
    this.pc.ontrack = (event) => {
      this.remoteStream = event.streams[0];
      document.getElementById('remoteVideo').srcObject = this.remoteStream;
    };
    
    return this.pc;
  }
  
  async call(userId) {
    this.remotePeer = userId;
    await this.startLocalMedia();
    this.createPeerConnection();
    
    const offer = await this.pc.createOffer();
    await this.pc.setLocalDescription(offer);
    
    this.socket.emit('offer', {
      to: userId,
      offer: offer
    });
  }
  
  async handleOffer(from, offer) {
    this.remotePeer = from;
    await this.startLocalMedia();
    this.createPeerConnection();
    
    await this.pc.setRemoteDescription(new RTCSessionDescription(offer));
    const answer = await this.pc.createAnswer();
    await this.pc.setLocalDescription(answer);
    
    this.socket.emit('answer', {
      to: from,
      answer: answer
    });
  }
  
  toggleMute() {
    this.isMuted = !this.isMuted;
    this.localStream.getAudioTracks().forEach(track => {
      track.enabled = !this.isMuted;
    });
    return this.isMuted;
  }
  
  toggleCamera() {
    this.isCameraOff = !this.isCameraOff;
    this.localStream.getVideoTracks().forEach(track => {
      track.enabled = !this.isCameraOff;
    });
    return this.isCameraOff;
  }
  
  async shareScreen() {
    const screenStream = await navigator.mediaDevices.getDisplayMedia({ video: true });
    const screenTrack = screenStream.getVideoTracks()[0];
    
    const sender = this.pc.getSenders().find(s => s.track?.kind === 'video');
    await sender.replaceTrack(screenTrack);
    
    screenTrack.onended = async () => {
      const cameraTrack = this.localStream.getVideoTracks()[0];
      await sender.replaceTrack(cameraTrack);
    };
  }
  
  hangUp() {
    this.socket.emit('hang-up', { to: this.remotePeer });
    this.endCall();
  }
  
  endCall() {
    // Stop all tracks
    this.localStream?.getTracks().forEach(track => track.stop());
    
    // Close peer connection
    this.pc?.close();
    this.pc = null;
    
    // Clear video elements
    document.getElementById('localVideo').srcObject = null;
    document.getElementById('remoteVideo').srcObject = null;
    
    this.onCallEnded?.();
  }
}

// Usage
const call = new VideoCall(socket);

call.onConnectionStateChange = (state) => {
  console.log('Call state:', state);
};

call.onCallEnded = () => {
  console.log('Call ended');
};

// Start call
await call.call('userId123');

// Controls
document.getElementById('muteBtn').onclick = () => {
  const muted = call.toggleMute();
  updateMuteButton(muted);
};

document.getElementById('cameraBtn').onclick = () => {
  const off = call.toggleCamera();
  updateCameraButton(off);
};

document.getElementById('screenBtn').onclick = () => {
  call.shareScreen();
};

document.getElementById('hangUpBtn').onclick = () => {
  call.hangUp();
};

Audio Processing and Analysis

// Use Web Audio API for advanced audio processing

async function createAudioProcessor(stream) {
  const audioContext = new AudioContext();
  const source = audioContext.createMediaStreamSource(stream);
  
  // Create analyzer for visualization
  const analyzer = audioContext.createAnalyser();
  analyzer.fftSize = 256;
  source.connect(analyzer);
  
  // Volume meter
  function getVolume() {
    const data = new Uint8Array(analyzer.frequencyBinCount);
    analyzer.getByteFrequencyData(data);
    const sum = data.reduce((a, b) => a + b, 0);
    return sum / data.length;
  }
  
  // Visualize
  function drawWaveform(canvas) {
    const ctx = canvas.getContext('2d');
    const data = new Uint8Array(analyzer.frequencyBinCount);
    
    function draw() {
      analyzer.getByteFrequencyData(data);
      
      ctx.fillStyle = 'rgb(0, 0, 0)';
      ctx.fillRect(0, 0, canvas.width, canvas.height);
      
      const barWidth = canvas.width / data.length;
      data.forEach((value, i) => {
        const height = (value / 255) * canvas.height;
        ctx.fillStyle = `hsl(${i * 2}, 100%, 50%)`;
        ctx.fillRect(i * barWidth, canvas.height - height, barWidth - 1, height);
      });
      
      requestAnimationFrame(draw);
    }
    draw();
  }
  
  return { getVolume, drawWaveform, analyzer };
}

// Noise gate (mute when not speaking)
function createNoiseGate(stream, threshold = 0.02) {
  const audioContext = new AudioContext();
  const source = audioContext.createMediaStreamSource(stream);
  const analyzer = audioContext.createAnalyser();
  const destination = audioContext.createMediaStreamDestination();
  const gainNode = audioContext.createGain();
  
  source.connect(analyzer);
  source.connect(gainNode);
  gainNode.connect(destination);
  
  analyzer.fftSize = 256;
  const data = new Float32Array(analyzer.fftSize);
  
  function checkLevel() {
    analyzer.getFloatTimeDomainData(data);
    const rms = Math.sqrt(data.reduce((sum, x) => sum + x * x, 0) / data.length);
    
    gainNode.gain.value = rms > threshold ? 1 : 0;
    requestAnimationFrame(checkLevel);
  }
  checkLevel();
  
  return destination.stream;
}