From c98e53b2677a5089597ca298e7f77aa06dacbaac Mon Sep 17 00:00:00 2001 From: Stuart Auld Date: Fri, 5 Jan 2024 14:11:12 +1100 Subject: [PATCH] Initial updates to persist transcoded files --- .github/workflows/build.yml | 62 +++++++++++----------- .github/workflows/docker.yml | 100 +++++++++++++++++------------------ .gitignore | 2 + hlsvod/manager.go | 72 ++++++++++++++++++++++++- hlsvod/types.go | 8 +-- internal/api/hlsvod.go | 46 ++++++++++++---- internal/config/config.go | 20 +++---- 7 files changed, 206 insertions(+), 104 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8308556..72145ff 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,38 +1,38 @@ -name: "CI for builds" +# name: "CI for builds" -on: - push: - tags: - - 'v*' +# on: +# push: +# tags: +# - 'v*' -jobs: - build-amd64: - name: amd64-${{ matrix.libc }} - runs-on: ubuntu-latest +# jobs: +# build-amd64: +# name: amd64-${{ matrix.libc }} +# runs-on: ubuntu-latest - strategy: - matrix: - include: - - container: golang:1.19-bullseye - libc: glibc - - container: golang:1.19-alpine - libc: musl +# strategy: +# matrix: +# include: +# - container: golang:1.19-bullseye +# libc: glibc +# - container: golang:1.19-alpine +# libc: musl - container: - image: ${{ matrix.container }} +# container: +# image: ${{ matrix.container }} - steps: - - name: Checkout - uses: actions/checkout@v3 +# steps: +# - name: Checkout +# uses: actions/checkout@v3 - - name: Build - run: go build -ldflags="-s -w" +# - name: Build +# run: go build -ldflags="-s -w" - - name: Upload to releases - uses: svenstaro/upload-release-action@v2 - id: attach_to_release - with: - file: go-transcode - asset_name: go-transcode-amd64-${{ matrix.libc }} - tag: ${{ github.ref }} - overwrite: true +# - name: Upload to releases +# uses: svenstaro/upload-release-action@v2 +# id: attach_to_release +# with: +# file: go-transcode +# asset_name: go-transcode-amd64-${{ matrix.libc }} +# tag: ${{ github.ref }} +# overwrite: true diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 6171c48..85eddce 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,53 +1,53 @@ -name: "CI for docker" +# name: "CI for docker" -on: - push: - branches: - - master - tags: - - 'v*' +# on: +# push: +# branches: +# - master +# tags: +# - 'v*' -jobs: - build-and-push-image: - runs-on: ubuntu-latest +# jobs: +# build-and-push-image: +# runs-on: ubuntu-latest - steps: - - - name: Checkout - uses: actions/checkout@v2 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v1 - - - name: Available platforms - run: echo ${{ steps.buildx.outputs.platforms }} - - - name: Extract metadata (tags, labels) for Docker - uses: docker/metadata-action@v3 - id: meta - with: - images: ghcr.io/${{ github.repository }} - tags: | - type=raw,value=latest,enable=${{ endsWith(github.ref, github.event.repository.default_branch) }} - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - - - name: Log in to the Container registry - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and push - uses: docker/build-push-action@v2 - with: - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - platforms: linux/amd64,linux/arm64,linux/arm/v7 - push: true +# steps: +# - +# name: Checkout +# uses: actions/checkout@v2 +# - +# name: Set up QEMU +# uses: docker/setup-qemu-action@v1 +# - +# name: Set up Docker Buildx +# id: buildx +# uses: docker/setup-buildx-action@v1 +# - +# name: Available platforms +# run: echo ${{ steps.buildx.outputs.platforms }} +# - +# name: Extract metadata (tags, labels) for Docker +# uses: docker/metadata-action@v3 +# id: meta +# with: +# images: ghcr.io/${{ github.repository }} +# tags: | +# type=raw,value=latest,enable=${{ endsWith(github.ref, github.event.repository.default_branch) }} +# type=semver,pattern={{version}} +# type=semver,pattern={{major}}.{{minor}} +# type=semver,pattern={{major}} +# - +# name: Log in to the Container registry +# uses: docker/login-action@v1 +# with: +# registry: ghcr.io +# username: ${{ github.actor }} +# password: ${{ secrets.GITHUB_TOKEN }} +# - +# name: Build and push +# uses: docker/build-push-action@v2 +# with: +# tags: ${{ steps.meta.outputs.tags }} +# labels: ${{ steps.meta.outputs.labels }} +# platforms: linux/amd64,linux/arm64,linux/arm/v7 +# push: true diff --git a/.gitignore b/.gitignore index 2eb24bd..9e8b89b 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ /go-transcode /bin /.env + +vids/ diff --git a/hlsvod/manager.go b/hlsvod/manager.go index 72fa284..c767f79 100644 --- a/hlsvod/manager.go +++ b/hlsvod/manager.go @@ -278,6 +278,36 @@ func (m *ManagerCtx) initialize() { Msg("initialization completed") } +func (m *ManagerCtx) loadPreexistingSegments() { + m.logger.Info().Msg("loading pre-existing segments") + + segments, err := os.ReadDir(m.config.TranscodeDir) + if err != nil && !os.IsNotExist(err) { + m.logger.Warn(). + Str("transcodeDir", m.config.TranscodeDir). + Msg("could not read directory") + return + } + + for _, s := range segments { + // Check if this is actually a ts chunk + if s.Name()[len(s.Name())-3:] != ".ts" { + m.logger.Warn().Str("segment", s.Name()).Msg("not an actual segment") + continue + } + + // Get the index + i, err := strconv.Atoi(strings.Split(strings.Split(s.Name(), "-")[1], ".")[0]) + if err != nil { + m.logger.Warn().Str("segment", s.Name()).Msg("not an actual segment") + continue + } + + m.logger.Debug().Str("segment", s.Name()).Msg("-loaded") + m.segments[i] = s.Name() + } +} + // // segments // @@ -296,6 +326,13 @@ func (m *ManagerCtx) getSegment(index int) (segmentPath string, ok bool) { segmentName, ok = m.segments[index] m.segmentsMu.RUnlock() + m.logger.Debug(). + Int("index", index). + Str("segmentName", segmentName). + Str("transcodeDir", m.config.TranscodeDir). + Bool("ok", ok). + Msg("getSegment") + if !ok { return } @@ -380,7 +417,11 @@ func (m *ManagerCtx) transcodeSegments(offset, limit int) error { logger := m.logger.With().Int("offset", offset).Int("limit", limit).Logger() segmentTimes := m.breakpoints[offset : offset+limit+1] - logger.Info().Interface("segments-times", segmentTimes).Msg("transcoding segments") + logger.Info(). + Str("outputDirPath", m.config.TranscodeDir). + Str("segmentPrefix", m.config.SegmentPrefix). + Interface("segments-times", segmentTimes). + Msg("transcoding segments") segments, err := TranscodeSegments(m.ctx, m.config.FFmpegBinary, TranscodeConfig{ InputFilePath: m.config.MediaPath, @@ -515,6 +556,11 @@ func (m *ManagerCtx) Start() (err error) { // initialization based on metadata m.initialize() + // load pre-existing segments + if m.config.PersistTranscodes { + m.loadPreexistingSegments() + } + // set ready state as done m.readyDone() }() @@ -530,7 +576,10 @@ func (m *ManagerCtx) Stop() { m.cancel() // remove all transcoded segments - m.clearAllSegments() + if !m.config.PersistTranscodes { + m.logger.Debug().Msg("clearing transcodes") + m.clearAllSegments() + } } func (m *ManagerCtx) Preload(ctx context.Context) (*ProbeMediaData, error) { @@ -574,6 +623,20 @@ func (m *ManagerCtx) ServeMedia(w http.ResponseWriter, r *http.Request) { return } + m.logger.Debug(). + Str("segmentPath", segmentPath). + Str("reqSegName", reqSegName). + Int("index", index). + Msg("serveMedia") + + // If running in persistent mode, check if we already have the segment before transcoding + // if m.config.PersistTranscodes { + // if _, err := os.Stat(segmentPath); os.IsNotExist(err) { + // serveExistingSegment(w, r, segmentPath) + // return + // } + // } + // try to transcode from current segment if err := m.transcodeFromSegment(index); err != nil { m.logger.Err(err).Int("index", index).Msg("unable to transcode media") @@ -623,7 +686,12 @@ func (m *ManagerCtx) ServeMedia(w http.ResponseWriter, r *http.Request) { } // return existing segment + serveExistingSegment(w, r, segmentPath) +} + +func serveExistingSegment(w http.ResponseWriter, r *http.Request, segmentPath string) { w.Header().Set("Content-Type", "application/vnd.apple.mpegurl") + // @TODO configure cache control w.Header().Set("Cache-Control", "no-cache") http.ServeFile(w, r, segmentPath) } diff --git a/hlsvod/types.go b/hlsvod/types.go index 7c4e651..7b26e66 100644 --- a/hlsvod/types.go +++ b/hlsvod/types.go @@ -6,9 +6,11 @@ import ( ) type Config struct { - MediaPath string // Transcoded video input. - TranscodeDir string // Temporary directory to store transcoded elements. - SegmentPrefix string + MediaPath string // Transcoded video input. + TranscodeDir string // Temporary directory to store transcoded elements. + TranscodeToInputPath bool // Stores the transcodes alongside the raw input + PersistTranscodes bool // Stores the transcodes in a permanent directory + SegmentPrefix string VideoProfile *VideoProfile VideoKeyframes bool diff --git a/internal/api/hlsvod.go b/internal/api/hlsvod.go index 39406b1..9481318 100644 --- a/internal/api/hlsvod.go +++ b/internal/api/hlsvod.go @@ -107,6 +107,7 @@ func (a *ApiManagerCtx) HlsVod(r chi.Router) { Str("path", urlPath). Str("hlsResource", hlsResource). Str("vodMediaPath", vodMediaPath). + Str("profileID", profileID). Msg("new hls vod request") // if manager was not found @@ -117,19 +118,46 @@ func (a *ApiManagerCtx) HlsVod(r chi.Router) { return } - // create own transcoding directory - transcodeDir, err := os.MkdirTemp(a.config.Vod.TranscodeDir, fmt.Sprintf("vod-%s-*", profileID)) - if err != nil { - logger.Warn().Err(err).Msg("could not create temp dir") - http.Error(w, "500 could not create temp dir", http.StatusInternalServerError) - return + // Check if we want to store transcodes alongside the original media + var transcodeSubDir string + if a.config.Vod.TranscodeToInputPath { + transcodeSubDir = filepath.Dir(vodMediaPath) + } else { + transcodeSubDir = a.config.Vod.TranscodeDir + } + + logger.Info(). + Bool("persistTranscodes", a.config.Vod.PersistTranscodes). + Str("transcodeDir", transcodeSubDir). + Msg("creating manager") + + var transcodeDir string + // create persistent directory if requested + if a.config.Vod.PersistTranscodes { + transcodeDir = filepath.Join(transcodeSubDir, fmt.Sprintf("vod-%s", profileID)) + err := os.Mkdir(transcodeDir, 0750) + if err != nil && !os.IsExist(err) { + logger.Warn().Err(err).Msg("could not create dir") + http.Error(w, "500 could not create dir", http.StatusInternalServerError) + return + } + } else { + // create temp transcoding directory + var err error + transcodeDir, err = os.MkdirTemp(transcodeSubDir, fmt.Sprintf("vod-%s-*", profileID)) + if err != nil { + logger.Warn().Err(err).Msg("could not create temp dir") + http.Error(w, "500 could not create temp dir", http.StatusInternalServerError) + return + } } // create new manager manager = hlsvod.New(hlsvod.Config{ - MediaPath: vodMediaPath, - TranscodeDir: transcodeDir, - SegmentPrefix: profileID, + MediaPath: vodMediaPath, + TranscodeDir: transcodeDir, + PersistTranscodes: a.config.Vod.PersistTranscodes, + SegmentPrefix: profileID, VideoProfile: &hlsvod.VideoProfile{ Width: profile.Width, diff --git a/internal/config/config.go b/internal/config/config.go index 49079af..fa58433 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -54,15 +54,17 @@ type AudioProfile struct { } type VOD struct { - MediaDir string `mapstructure:"media-dir"` - TranscodeDir string `mapstructure:"transcode-dir"` - VideoProfiles map[string]VideoProfile `mapstructure:"video-profiles"` - VideoKeyframes bool `mapstructure:"video-keyframes"` - AudioProfile AudioProfile `mapstructure:"audio-profile"` - Cache bool `mapstructure:"cache"` - CacheDir string `mapstructure:"cache-dir"` - FFmpegBinary string `mapstructure:"ffmpeg-binary"` - FFprobeBinary string `mapstructure:"ffprobe-binary"` + MediaDir string `mapstructure:"media-dir"` + TranscodeDir string `mapstructure:"transcode-dir"` + TranscodeToInputPath bool `mapstructure:"transcode-to-input-path"` + PersistTranscodes bool `mapstructure:"persist-transcodes"` + VideoProfiles map[string]VideoProfile `mapstructure:"video-profiles"` + VideoKeyframes bool `mapstructure:"video-keyframes"` + AudioProfile AudioProfile `mapstructure:"audio-profile"` + Cache bool `mapstructure:"cache"` + CacheDir string `mapstructure:"cache-dir"` + FFmpegBinary string `mapstructure:"ffmpeg-binary"` + FFprobeBinary string `mapstructure:"ffprobe-binary"` } type Enigma2 struct {