现在合成的有问题，时间轴错乱，在房间中有多个用户情况下

2026-06-02 23:59:55 +08:00
parent 206a3ac91d
commit 3e161ff995
4 changed files with 298 additions and 22 deletions
--- a/src/recording/composer.ts
+++ b/src/recording/composer.ts
@@ -53,6 +53,12 @@ type CompositionInputSets = {
  audioInputs: ServerTrackRecordingFile[];
 };

+type VideoTimelineSegment = {
+  startMs: number;
+  endMs: number;
+  activeInputs: ServerTrackRecordingFile[];
+};
+
 const jobs: Map<string, RecordingCompositionJob> = new Map<string, RecordingCompositionJob>();

 function nowIso(): string {
@@ -120,6 +126,87 @@ function orderVideoInputsForComposition(files: ServerTrackRecordingFile[]): Serv
  ];
 }

+function parseInputTimestamp(value: unknown): number | null {
+  if (typeof value !== 'string' || !value.trim()) {
+    return null;
+  }
+
+  const timestamp = Date.parse(value);
+  return isFinite(timestamp) ? timestamp : null;
+}
+
+function getInputStartMs(file: ServerTrackRecordingFile): number | null {
+  const metadata = file.metadata || {};
+  return parseInputTimestamp(metadata.recordingStartedAt)
+    || parseInputTimestamp(file.recordingStartedAt)
+    || parseInputTimestamp(metadata.startedAt)
+    || parseInputTimestamp(file.uploadedAt)
+    || parseInputTimestamp(metadata.uploadedAt);
+}
+
+function getInputEndMs(file: ServerTrackRecordingFile): number | null {
+  const metadata = file.metadata || {};
+  return parseInputTimestamp(metadata.recordingEndedAt)
+    || parseInputTimestamp(file.recordingEndedAt)
+    || parseInputTimestamp(metadata.endedAt)
+    || parseInputTimestamp(metadata.updatedAt);
+}
+
+function getTimelineOriginMs(files: ServerTrackRecordingFile[]): number | null {
+  const starts = files
+    .map(getInputStartMs)
+    .filter((timestamp) => timestamp !== null) as number[];
+  if (starts.length === 0) {
+    return null;
+  }
+
+  return Math.min(...starts);
+}
+
+function getTimelineDurationSeconds(files: ServerTrackRecordingFile[], timelineOriginMs: number | null): number | null {
+  if (timelineOriginMs === null) {
+    return null;
+  }
+
+  const ends = files
+    .map(getInputEndMs)
+    .filter((timestamp) => timestamp !== null) as number[];
+  if (ends.length === 0) {
+    return null;
+  }
+
+  const durationSeconds = (Math.max(...ends) - timelineOriginMs) / 1000;
+  return durationSeconds > 0 ? durationSeconds : null;
+}
+
+function getTimelineEndMs(files: ServerTrackRecordingFile[]): number | null {
+  const ends = files
+    .map(getInputEndMs)
+    .filter((timestamp) => timestamp !== null) as number[];
+  if (ends.length === 0) {
+    return null;
+  }
+
+  return Math.max(...ends);
+}
+
+function getInputOffsetSeconds(file: ServerTrackRecordingFile, timelineOriginMs: number | null): number {
+  const startMs = getInputStartMs(file);
+  if (startMs === null || timelineOriginMs === null) {
+    return 0;
+  }
+
+  return Math.max(0, (startMs - timelineOriginMs) / 1000);
+}
+
+function formatSeconds(value: number): string {
+  if (value <= 0.001) {
+    return '0';
+  }
+
+  return value.toFixed(3).replace(/\.?0+$/, '');
+}
+
 function getBottomTileWidth(index: number, inputCount: number, outputWidth: number): number {
  const sideCount = inputCount - 1;
  if (sideCount <= 1) {
@@ -142,6 +229,62 @@ function createHostBottomLayout(inputCount: number, outputWidth: number, hostHei
  return positions.join('|');
 }

+function sortActiveVideoInputs(files: ServerTrackRecordingFile[]): ServerTrackRecordingFile[] {
+  return orderVideoInputsForComposition(files).sort((a, b) => {
+    const aIsHost = isHostInput(a);
+    const bIsHost = isHostInput(b);
+    if (aIsHost !== bIsHost) {
+      return aIsHost ? -1 : 1;
+    }
+    return a.participantId.localeCompare(b.participantId);
+  });
+}
+
+function getVideoTimelineSegments(
+  files: ServerTrackRecordingFile[],
+  timelineOriginMs: number | null,
+  timelineEndMs: number | null
+): VideoTimelineSegment[] {
+  if (timelineOriginMs === null || timelineEndMs === null || timelineEndMs <= timelineOriginMs) {
+    return [];
+  }
+
+  const pointsByMs: { [timestamp: string]: boolean } = {};
+  pointsByMs[String(timelineOriginMs)] = true;
+  pointsByMs[String(timelineEndMs)] = true;
+  files.forEach((file) => {
+    const startMs = getInputStartMs(file);
+    const endMs = getInputEndMs(file);
+    if (startMs !== null && startMs > timelineOriginMs && startMs < timelineEndMs) {
+      pointsByMs[String(startMs)] = true;
+    }
+    if (endMs !== null && endMs > timelineOriginMs && endMs < timelineEndMs) {
+      pointsByMs[String(endMs)] = true;
+    }
+  });
+
+  const points = Object.keys(pointsByMs)
+    .map((value) => Number(value))
+    .sort((a, b) => a - b);
+  const segments: VideoTimelineSegment[] = [];
+  for (let index = 0; index < points.length - 1; index += 1) {
+    const startMs = points[index];
+    const endMs = points[index + 1];
+    if (endMs <= startMs) {
+      continue;
+    }
+
+    const activeInputs = sortActiveVideoInputs(files.filter((file) => {
+      const inputStartMs = getInputStartMs(file) || timelineOriginMs;
+      const inputEndMs = getInputEndMs(file) || timelineEndMs;
+      return inputStartMs < endMs && inputEndMs > startMs;
+    }));
+    segments.push({ startMs, endMs, activeInputs });
+  }
+
+  return segments;
+}
+
 export function buildFfmpegCompositionArgs(input: {
  videoInputs: ServerTrackRecordingFile[];
  audioInputs: ServerTrackRecordingFile[];
@@ -153,6 +296,10 @@ export function buildFfmpegCompositionArgs(input: {
  const hostHeight = 540;
  const bottomHeight = outputHeight - hostHeight;
  const videoInputs = orderVideoInputsForComposition(input.videoInputs);
+  const timelineOriginMs = getTimelineOriginMs(videoInputs.concat(input.audioInputs));
+  const timelineEndMs = getTimelineEndMs(videoInputs.concat(input.audioInputs));
+  const timelineDurationSeconds = getTimelineDurationSeconds(videoInputs.concat(input.audioInputs), timelineOriginMs);
+  const videoSegments = getVideoTimelineSegments(videoInputs, timelineOriginMs, timelineEndMs);
  const args = ['-y'];
  const orderedInputs = videoInputs.concat(input.audioInputs);
  orderedInputs.forEach((file) => {
@@ -160,30 +307,72 @@ export function buildFfmpegCompositionArgs(input: {
  });

  const filters: string[] = [];
-  videoInputs.forEach((_file, index) => {
-    const width = videoInputs.length === 1
-      ? outputWidth
-      : index === 0 ? outputWidth : getBottomTileWidth(index - 1, videoInputs.length, outputWidth);
-    const height = videoInputs.length === 1
-      ? outputHeight
-      : index === 0 ? hostHeight : bottomHeight;
-    filters.push(`[${index}:v]scale=${width}:${height}:force_original_aspect_ratio=decrease,pad=${width}:${height}:(ow-iw)/2:(oh-ih)/2:black,setsar=1[v${index}]`);
+  const videoInputUseCounts = videoInputs.map((file) => videoSegments.filter((segment) => segment.activeInputs.indexOf(file) >= 0).length);
+  const videoInputUsePositions = videoInputs.map(() => 0);
+  videoInputUseCounts.forEach((useCount, inputIndex) => {
+    if (useCount <= 1) {
+      return;
+    }
+
+    const splitLabels = [];
+    for (let splitIndex = 0; splitIndex < useCount; splitIndex += 1) {
+      splitLabels.push(`[vin${inputIndex}_${splitIndex}]`);
+    }
+    filters.push(`[${inputIndex}:v]split=${useCount}${splitLabels.join('')}`);
  });

-  if (videoInputs.length === 1) {
-    filters.push('[v0]fps=30,format=yuv420p[vout]');
+  videoSegments.forEach((segment, segmentIndex) => {
+    const segmentDurationSeconds = (segment.endMs - segment.startMs) / 1000;
+    if (segment.activeInputs.length === 0) {
+      filters.push(`color=color=black:size=${outputWidth}x${outputHeight}:rate=30:duration=${formatSeconds(segmentDurationSeconds)},format=yuv420p[seg${segmentIndex}]`);
+      return;
+    }
+
+    segment.activeInputs.forEach((file, activeIndex) => {
+      const inputIndex = videoInputs.indexOf(file);
+      const inputLabel = videoInputUseCounts[inputIndex] > 1
+        ? `vin${inputIndex}_${videoInputUsePositions[inputIndex]++}`
+        : `${inputIndex}:v`;
+      const inputStartMs = getInputStartMs(file) || segment.startMs;
+      const trimStartSeconds = Math.max(0, (segment.startMs - inputStartMs) / 1000);
+      const width = segment.activeInputs.length === 1
+        ? outputWidth
+        : activeIndex === 0 ? outputWidth : getBottomTileWidth(activeIndex - 1, segment.activeInputs.length, outputWidth);
+      const height = segment.activeInputs.length === 1
+        ? outputHeight
+        : activeIndex === 0 ? hostHeight : bottomHeight;
+      filters.push(`[${inputLabel}]trim=start=${formatSeconds(trimStartSeconds)}:duration=${formatSeconds(segmentDurationSeconds)},setpts=PTS-STARTPTS,scale=${width}:${height}:force_original_aspect_ratio=decrease,pad=${width}:${height}:(ow-iw)/2:(oh-ih)/2:black,setsar=1[seg${segmentIndex}v${activeIndex}]`);
+    });
+
+    if (segment.activeInputs.length === 1) {
+      filters.push(`[seg${segmentIndex}v0]fps=30,format=yuv420p[seg${segmentIndex}]`);
+      return;
+    }
+
+    const segmentVideoLabels = segment.activeInputs.map((_file, activeIndex) => `[seg${segmentIndex}v${activeIndex}]`).join('');
+    filters.push(`${segmentVideoLabels}xstack=inputs=${segment.activeInputs.length}:layout=${createHostBottomLayout(segment.activeInputs.length, outputWidth, hostHeight)}:fill=black,fps=30,format=yuv420p[seg${segmentIndex}]`);
+  });
+
+  if (videoSegments.length === 1) {
+    filters.push('[seg0]null[vout]');
  } else {
-    const videoLabels = videoInputs.map((_file, index) => `[v${index}]`).join('');
-    filters.push(`${videoLabels}xstack=inputs=${videoInputs.length}:layout=${createHostBottomLayout(videoInputs.length, outputWidth, hostHeight)}:fill=black,fps=30,format=yuv420p[vout]`);
+    const videoLabels = videoSegments.map((_segment, index) => `[seg${index}]`).join('');
+    filters.push(`${videoLabels}concat=n=${videoSegments.length}:v=1:a=0[vout]`);
  }

  if (input.audioInputs.length === 1) {
    const audioInputIndex = videoInputs.length;
-    filters.push(`[${audioInputIndex}:a]aresample=async=1:first_pts=0[aout]`);
+    const offsetMs = Math.round(getInputOffsetSeconds(input.audioInputs[0], timelineOriginMs) * 1000);
+    const offsetFilter = offsetMs > 1 ? `,adelay=${offsetMs}:all=1` : '';
+    filters.push(`[${audioInputIndex}:a]aresample=async=1:first_pts=0${offsetFilter}[aout]`);
  } else if (input.audioInputs.length > 1) {
-    const audioLabels = input.audioInputs
-      .map((_file, index) => `[${videoInputs.length + index}:a]`)
-      .join('');
+    const audioLabels = input.audioInputs.map((file, index) => {
+      const audioInputIndex = videoInputs.length + index;
+      const offsetMs = Math.round(getInputOffsetSeconds(file, timelineOriginMs) * 1000);
+      const offsetFilter = offsetMs > 1 ? `,adelay=${offsetMs}:all=1` : '';
+      filters.push(`[${audioInputIndex}:a]aresample=async=1:first_pts=0${offsetFilter}[a${index}]`);
+      return `[a${index}]`;
+    }).join('');
    filters.push(`${audioLabels}amix=inputs=${input.audioInputs.length}:duration=longest:dropout_transition=2[aout]`);
  }

@@ -204,7 +393,10 @@ export function buildFfmpegCompositionArgs(input: {
    }
  }

-  args.push('-shortest', input.outputPath);
+  if (timelineDurationSeconds !== null) {
+    args.push('-t', formatSeconds(timelineDurationSeconds));
+  }
+  args.push(input.outputPath);
  return args;
 }

--- a/src/recording/storage.ts
+++ b/src/recording/storage.ts
@@ -15,6 +15,8 @@ export type ServerTrackRecordingFile = ServerTrackRecordingTarget & {
  trackId: string;
  trackKind: string;
  uploadedAt: string;
+  recordingStartedAt?: string;
+  recordingEndedAt?: string;
  metadata: any;
 };

@@ -143,6 +145,7 @@ export function writeServerTrackRecordingMetadata(input: WriteMetadataInput): vo
    ],
    uploadedAt: now,
    updatedAt: now,
+    recordingStartedAt: now,
    recordingSource: 'server',
    recordingId: input.recordingId,
    participantId: input.participantId,
@@ -162,6 +165,7 @@ export function updateServerTrackRecordingMetadataSize(target: ServerTrackRecord
  const metadata = JSON.parse(fs.readFileSync(target.metadataPath, 'utf8'));
  metadata.size = fs.statSync(target.filePath).size;
  metadata.updatedAt = new Date().toISOString();
+  metadata.recordingEndedAt = metadata.updatedAt;
  fs.writeFileSync(target.metadataPath, JSON.stringify(metadata, null, 2));
 }

@@ -208,7 +212,9 @@ export function listServerTrackRecordingFiles(input: {
        participantId: metadata.participantId || '',
        trackId: metadata.trackId || '',
        trackKind: metadata.trackKind || '',
-        uploadedAt: metadata.uploadedAt || fs.statSync(filePath).birthtime.toISOString()
+        uploadedAt: metadata.uploadedAt || fs.statSync(filePath).birthtime.toISOString(),
+        recordingStartedAt: metadata.recordingStartedAt || metadata.uploadedAt,
+        recordingEndedAt: metadata.recordingEndedAt || metadata.updatedAt
      };
    })
    .filter((file) => Boolean(file)) as ServerTrackRecordingFile[];
--- a/test/recording-composer.test.ts
+++ b/test/recording-composer.test.ts
@@ -1,7 +1,14 @@
 import { buildFfmpegCompositionArgs } from '../src/recording/composer';
 import { ServerTrackRecordingFile } from '../src/recording/storage';

-function file(filename: string, trackKind: string, participantId: string, role = 'participant'): ServerTrackRecordingFile {
+function file(
+  filename: string,
+  trackKind: string,
+  participantId: string,
+  role = 'participant',
+  recordingStartedAt = '2026-06-01T00:00:00.000Z',
+  recordingEndedAt = '2026-06-01T00:00:10.000Z'
+): ServerTrackRecordingFile {
  return {
    meetingId: 'room-1',
    directory: 'recordings/room-1',
@@ -12,8 +19,10 @@ function file(filename: string, trackKind: string, participantId: string, role =
    participantId,
    trackId: `${participantId}-${trackKind}`,
    trackKind,
-    uploadedAt: '2026-06-01T00:00:00.000Z',
-    metadata: { role }
+    uploadedAt: recordingStartedAt,
+    recordingStartedAt,
+    recordingEndedAt,
+    metadata: { role, recordingStartedAt, recordingEndedAt, updatedAt: recordingEndedAt }
  };
 }

@@ -40,6 +49,7 @@ describe('recording composer', () => {
    expect(args.join(' ')).toContain('amix=inputs=2');
    expect(args).toContain('libvpx-vp9');
    expect(args).toContain('libopus');
+    expect(args).not.toContain('-shortest');
    expect(args[args.length - 1]).toBe('recordings/room-1/output.webm');
  });

@@ -82,4 +92,70 @@ describe('recording composer', () => {
    expect(args).toContain('-pix_fmt');
    expect(args).not.toContain('libopus');
  });
+
+  test('pads late participant tracks to keep the room timeline aligned', () => {
+    const args = buildFfmpegCompositionArgs({
+      videoInputs: [
+        file('host-video.webm', 'video', 'host', 'host', '2026-06-01T00:00:00.000Z', '2026-06-01T00:00:10.000Z'),
+        file('p1-video.webm', 'video', 'p1', 'participant', '2026-06-01T00:00:02.500Z', '2026-06-01T00:00:10.000Z')
+      ],
+      audioInputs: [
+        file('host-audio.webm', 'audio', 'host', 'host', '2026-06-01T00:00:00.000Z', '2026-06-01T00:00:10.000Z'),
+        file('p1-audio.webm', 'audio', 'p1', 'participant', '2026-06-01T00:00:02.500Z', '2026-06-01T00:00:10.000Z')
+      ],
+      outputPath: 'recordings/room-1/output.webm',
+      format: 'webm'
+    });
+
+    const filter = args[args.indexOf('-filter_complex') + 1];
+    expect(filter).toContain('[0:v]split=2[vin0_0][vin0_1]');
+    expect(filter).toContain('[vin0_0]trim=start=0:duration=2.5');
+    expect(filter).toContain('[vin0_1]trim=start=2.5:duration=7.5');
+    expect(filter).toContain('[1:v]trim=start=0:duration=7.5');
+    expect(filter).toContain('concat=n=2:v=1:a=0[vout]');
+    expect(filter).toContain('[2:a]aresample=async=1:first_pts=0[a0]');
+    expect(filter).toContain('[3:a]aresample=async=1:first_pts=0,adelay=2500:all=1[a1]');
+    expect(filter).toContain('[a0][a1]amix=inputs=2:duration=longest');
+  });
+
+  test('changes the layout when participants join and leave without overlapping', () => {
+    const args = buildFfmpegCompositionArgs({
+      videoInputs: [
+        file('host-video.webm', 'video', 'host', 'host', '2026-06-01T00:00:00.000Z', '2026-06-01T00:00:12.000Z'),
+        file('p1-video.webm', 'video', 'p1', 'participant', '2026-06-01T00:00:00.000Z', '2026-06-01T00:00:05.000Z'),
+        file('p2-video.webm', 'video', 'p2', 'participant', '2026-06-01T00:00:05.000Z', '2026-06-01T00:00:12.000Z')
+      ],
+      audioInputs: [],
+      outputPath: 'recordings/room-1/output.webm',
+      format: 'webm'
+    });
+
+    const filter = args[args.indexOf('-filter_complex') + 1];
+    expect(filter).toContain('xstack=inputs=2');
+    expect(filter).toContain('layout=0_0|0_540');
+    expect(filter).toContain('[0:v]split=2[vin0_0][vin0_1]');
+    expect(filter).toContain('[vin0_0]trim=start=0:duration=5');
+    expect(filter).toContain('[1:v]trim=start=0:duration=5');
+    expect(filter).toContain('[vin0_1]trim=start=5:duration=7');
+    expect(filter).toContain('[2:v]trim=start=0:duration=7');
+    expect(filter).toContain('concat=n=2:v=1:a=0[vout]');
+    expect(filter).not.toContain('xstack=inputs=3');
+  });
+
+  test('keeps separate viewports for participants whose video intervals overlap', () => {
+    const args = buildFfmpegCompositionArgs({
+      videoInputs: [
+        file('host-video.webm', 'video', 'host', 'host', '2026-06-01T00:00:00.000Z', '2026-06-01T00:00:12.000Z'),
+        file('p1-video.webm', 'video', 'p1', 'participant', '2026-06-01T00:00:00.000Z', '2026-06-01T00:00:08.000Z'),
+        file('p2-video.webm', 'video', 'p2', 'participant', '2026-06-01T00:00:05.000Z', '2026-06-01T00:00:12.000Z')
+      ],
+      audioInputs: [],
+      outputPath: 'recordings/room-1/output.webm',
+      format: 'webm'
+    });
+
+    const filter = args[args.indexOf('-filter_complex') + 1];
+    expect(filter).toContain('xstack=inputs=3');
+    expect(filter).toContain('layout=0_0|0_540|640_540');
+  });
 });
--- a/test/recording-storage.test.ts
+++ b/test/recording-storage.test.ts
@@ -64,7 +64,9 @@ describe('recording storage', () => {
      userId: 'server-recorder',
      recordingSource: 'server',
      participantId: 'participant-1',
-      trackKind: 'video'
+      trackKind: 'video',
+      recordingStartedAt: expect.any(String),
+      recordingEndedAt: expect.any(String)
    }));
    const files = listServerTrackRecordingFiles({
      meetingId: 'room_1',