Segmenting from a buffer back into a buffer for upload

[stef-coenen]

We are trying to build an app with support for streaming fragmented mp4's directly from the server. In order to do so we are looking to segment an mp4 during the upload. The goal is to get a segmented byteArray from a readable stream. We currently have the following, but the moov box is incorrect and we can't get the audio track included.

Any clues, where to look? The documentation only got us so far, and the library has many more features, so we are confident is has to be possible in some way.

        let segmentedBytes: Uint8Array;

        mp4File.onError = function (e: unknown) {
            reject(e);
        };

        mp4File.onReady = function (info: { tracks: { id: number; nb_samples: number }[] }) {
            // TODO: Fix audio track.. it's not working
            // for (var i = 0; i < info.tracks.length; i++) {
            for (var i = 0; i < 1; i++) {
                var track = info.tracks[i];
                mp4File.setSegmentOptions(track.id, null, {
                    nb_samples: info.tracks[i].nb_samples,
                });
            }
            const initSegment: { id: number; buffer: ArrayBuffer }[] =
                mp4File.initializeSegmentation();

            segmentedBytes = mergeByteArrays(initSegment.map((s) => new Uint8Array(s.buffer)));
            mp4File.start();
        };

        mp4File.onSegment = function (
            id: number,
            user: unknown,
            buffer: ArrayBuffer,
            sampleNum: number,
            is_last: boolean
        ) {
            console.debug(
                `Recieved ${
                    is_last ? ' last' : ''
                } segment on track ${id} with sample up to ${sampleNum}`
            );

            segmentedBytes = mergeByteArrays([
                ...(segmentedBytes ? [segmentedBytes] : []),
                new Uint8Array(buffer),
            ]);

            if (is_last) {
                console.log({ mp4File });
                resolve(segmentedBytes);
            }
        };

        var offset = 0;
        var reader = file.stream().getReader();

        const getNextChunk = ({ done, value }: ReadableStreamReadResult<Uint8Array>): any => {
            if (done) {
                mp4File.stop();
                mp4File.flush();
                return;
            }

            const block: any = value.buffer;
            block.fileStart = offset;
            offset += value.length;

            mp4File.appendBuffer(block);
            return reader.read().then(getNextChunk);
        };

        reader.read().then(getNextChunk);```

[hughfenghen]

https://github.com/hughfenghen/WebAV/blob/main/packages/av-canvas/src/mp4-utils.ts#L364
Maybe you can refer to stream2file and file2stream two functions, you can read data from the stream returned by file2stream and upload it to the server.

export function file2stream (
  file,
  timeSlice
) {
  let timerId = 0

  let sendedBoxIdx = 0
  const boxes = file.boxes
  const deltaBuf = (): ArrayBuffer => {
    const ds = new mp4box.DataStream()
    ds.endianness = mp4box.DataStream.BIG_ENDIAN
    for (let i = sendedBoxIdx; i < boxes.length; i++) {
      boxes[i].write(ds)
    }
    sendedBoxIdx = boxes.length
    return ds.buffer
  }

  let stoped = false
  let exit = null
  const stream = new ReadableStream({
    start (ctrl) {
      timerId = self.setInterval(() => {
        ctrl.enqueue(deltaBuf())
      }, timeSlice)

      exit = () => {
        clearInterval(timerId)
        file.flush()
        ctrl.enqueue(deltaBuf())
        ctrl.close()
      }

      if (stoped) exit()
    },
    cancel () {
      clearInterval(timerId)
    }
  })

  return {
    stream,
    stop: () => {
      stoped = true
      exit?.()
    }
  }
}

This repository code is still unstable, maybe in the future it will packaged an MP4 + WebCodecs tool and release it to npm.