Initial updates to persist transcoded files

.github/workflows/build.yml

@@ -1,38 +1,38 @@
-name: "CI for builds"
+# name: "CI for builds"
 
-on:
-  push:
-    tags:
-      - 'v*'
+# on:
+#   push:
+#     tags:
+#       - 'v*'
 
-jobs:
-  build-amd64:
-    name: amd64-${{ matrix.libc }}
-    runs-on: ubuntu-latest
+# jobs:
+#   build-amd64:
+#     name: amd64-${{ matrix.libc }}
+#     runs-on: ubuntu-latest
 
-    strategy:
-      matrix:
-        include:
-          - container: golang:1.19-bullseye
-            libc: glibc
-          - container: golang:1.19-alpine
-            libc: musl
+#     strategy:
+#       matrix:
+#         include:
+#           - container: golang:1.19-bullseye
+#             libc: glibc
+#           - container: golang:1.19-alpine
+#             libc: musl
 
-    container:
-      image: ${{ matrix.container }}
+#     container:
+#       image: ${{ matrix.container }}
 
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
+#     steps:
+#       - name: Checkout
+#         uses: actions/checkout@v3
 
-      - name: Build
-        run: go build -ldflags="-s -w"
+#       - name: Build
+#         run: go build -ldflags="-s -w"
 
-      - name: Upload to releases
-        uses: svenstaro/upload-release-action@v2
-        id: attach_to_release
-        with:
-          file: go-transcode
-          asset_name: go-transcode-amd64-${{ matrix.libc }}
-          tag: ${{ github.ref }}
-          overwrite: true
+#       - name: Upload to releases
+#         uses: svenstaro/upload-release-action@v2
+#         id: attach_to_release
+#         with:
+#           file: go-transcode
+#           asset_name: go-transcode-amd64-${{ matrix.libc }}
+#           tag: ${{ github.ref }}
+#           overwrite: true

.github/workflows/docker.yml

@@ -1,53 +1,53 @@
-name: "CI for docker"
+# name: "CI for docker"
 
-on:
-  push:
-    branches:
-      - master
-    tags:
-      - 'v*'
+# on:
+#   push:
+#     branches:
+#       - master
+#     tags:
+#       - 'v*'
 
-jobs:
-  build-and-push-image:
-    runs-on: ubuntu-latest
+# jobs:
+#   build-and-push-image:
+#     runs-on: ubuntu-latest
 
-    steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v2
-      -
-        name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-      -
-        name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-      -
-        name: Available platforms
-        run: echo ${{ steps.buildx.outputs.platforms }}
-      - 
-        name: Extract metadata (tags, labels) for Docker
-        uses: docker/metadata-action@v3
-        id: meta
-        with:
-          images: ghcr.io/${{ github.repository }}
-          tags: |
-            type=raw,value=latest,enable=${{ endsWith(github.ref, github.event.repository.default_branch) }}
-            type=semver,pattern={{version}}
-            type=semver,pattern={{major}}.{{minor}}
-            type=semver,pattern={{major}}
-      - 
-        name: Log in to the Container registry
-        uses: docker/login-action@v1
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-      - 
-        name: Build and push
-        uses: docker/build-push-action@v2
-        with:
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-          platforms: linux/amd64,linux/arm64,linux/arm/v7
-          push: true
+#     steps:
+#       -
+#         name: Checkout
+#         uses: actions/checkout@v2
+#       -
+#         name: Set up QEMU
+#         uses: docker/setup-qemu-action@v1
+#       -
+#         name: Set up Docker Buildx
+#         id: buildx
+#         uses: docker/setup-buildx-action@v1
+#       -
+#         name: Available platforms
+#         run: echo ${{ steps.buildx.outputs.platforms }}
+#       -
+#         name: Extract metadata (tags, labels) for Docker
+#         uses: docker/metadata-action@v3
+#         id: meta
+#         with:
+#           images: ghcr.io/${{ github.repository }}
+#           tags: |
+#             type=raw,value=latest,enable=${{ endsWith(github.ref, github.event.repository.default_branch) }}
+#             type=semver,pattern={{version}}
+#             type=semver,pattern={{major}}.{{minor}}
+#             type=semver,pattern={{major}}
+#       -
+#         name: Log in to the Container registry
+#         uses: docker/login-action@v1
+#         with:
+#           registry: ghcr.io
+#           username: ${{ github.actor }}
+#           password: ${{ secrets.GITHUB_TOKEN }}
+#       -
+#         name: Build and push
+#         uses: docker/build-push-action@v2
+#         with:
+#           tags: ${{ steps.meta.outputs.tags }}
+#           labels: ${{ steps.meta.outputs.labels }}
+#           platforms: linux/amd64,linux/arm64,linux/arm/v7
+#           push: true

.gitignore

@@ -3,3 +3,5 @@
 /go-transcode
 /bin
 /.env
+
+vids/

hlsvod/manager.go

@@ -278,6 +278,36 @@ func (m *ManagerCtx) initialize() {
 		Msg("initialization completed")
 }
 
+func (m *ManagerCtx) loadPreexistingSegments() {
+	m.logger.Info().Msg("loading pre-existing segments")
+
+	segments, err := os.ReadDir(m.config.TranscodeDir)
+	if err != nil && !os.IsNotExist(err) {
+		m.logger.Warn().
+			Str("transcodeDir", m.config.TranscodeDir).
+			Msg("could not read directory")
+		return
+	}
+
+	for _, s := range segments {
+		// Check if this is actually a ts chunk
+		if s.Name()[len(s.Name())-3:] != ".ts" {
+			m.logger.Warn().Str("segment", s.Name()).Msg("not an actual segment")
+			continue
+		}
+
+		// Get the index
+		i, err := strconv.Atoi(strings.Split(strings.Split(s.Name(), "-")[1], ".")[0])
+		if err != nil {
+			m.logger.Warn().Str("segment", s.Name()).Msg("not an actual segment")
+			continue
+		}
+
+		m.logger.Debug().Str("segment", s.Name()).Msg("-loaded")
+		m.segments[i] = s.Name()
+	}
+}
+
 //
 // segments
 //
@@ -296,6 +326,13 @@ func (m *ManagerCtx) getSegment(index int) (segmentPath string, ok bool) {
 	segmentName, ok = m.segments[index]
 	m.segmentsMu.RUnlock()
 
+	m.logger.Debug().
+		Int("index", index).
+		Str("segmentName", segmentName).
+		Str("transcodeDir", m.config.TranscodeDir).
+		Bool("ok", ok).
+		Msg("getSegment")
+
 	if !ok {
 		return
 	}
@@ -380,7 +417,11 @@ func (m *ManagerCtx) transcodeSegments(offset, limit int) error {
 	logger := m.logger.With().Int("offset", offset).Int("limit", limit).Logger()
 
 	segmentTimes := m.breakpoints[offset : offset+limit+1]
-	logger.Info().Interface("segments-times", segmentTimes).Msg("transcoding segments")
+	logger.Info().
+		Str("outputDirPath", m.config.TranscodeDir).
+		Str("segmentPrefix", m.config.SegmentPrefix).
+		Interface("segments-times", segmentTimes).
+		Msg("transcoding segments")
 
 	segments, err := TranscodeSegments(m.ctx, m.config.FFmpegBinary, TranscodeConfig{
 		InputFilePath: m.config.MediaPath,
@@ -515,6 +556,11 @@ func (m *ManagerCtx) Start() (err error) {
 		// initialization based on metadata
 		m.initialize()
 
+		// load pre-existing segments
+		if m.config.PersistTranscodes {
+			m.loadPreexistingSegments()
+		}
+
 		// set ready state as done
 		m.readyDone()
 	}()
@@ -530,7 +576,10 @@ func (m *ManagerCtx) Stop() {
 	m.cancel()
 
 	// remove all transcoded segments
-	m.clearAllSegments()
+	if !m.config.PersistTranscodes {
+		m.logger.Debug().Msg("clearing transcodes")
+		m.clearAllSegments()
+	}
 }
 
 func (m *ManagerCtx) Preload(ctx context.Context) (*ProbeMediaData, error) {
@@ -574,6 +623,20 @@ func (m *ManagerCtx) ServeMedia(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	m.logger.Debug().
+		Str("segmentPath", segmentPath).
+		Str("reqSegName", reqSegName).
+		Int("index", index).
+		Msg("serveMedia")
+
+	// If running in persistent mode, check if we already have the segment before transcoding
+	// if m.config.PersistTranscodes {
+	// 	if _, err := os.Stat(segmentPath); os.IsNotExist(err) {
+	// 		serveExistingSegment(w, r, segmentPath)
+	// 		return
+	// 	}
+	// }
+
 	// try to transcode from current segment
 	if err := m.transcodeFromSegment(index); err != nil {
 		m.logger.Err(err).Int("index", index).Msg("unable to transcode media")
@@ -623,7 +686,12 @@ func (m *ManagerCtx) ServeMedia(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// return existing segment
+	serveExistingSegment(w, r, segmentPath)
+}
+
+func serveExistingSegment(w http.ResponseWriter, r *http.Request, segmentPath string) {
 	w.Header().Set("Content-Type", "application/vnd.apple.mpegurl")
+	// @TODO configure cache control
 	w.Header().Set("Cache-Control", "no-cache")
 	http.ServeFile(w, r, segmentPath)
 }

hlsvod/types.go

@@ -6,9 +6,11 @@ import (
 )
 
 type Config struct {
-	MediaPath     string // Transcoded video input.
-	TranscodeDir  string // Temporary directory to store transcoded elements.
-	SegmentPrefix string
+	MediaPath            string // Transcoded video input.
+	TranscodeDir         string // Temporary directory to store transcoded elements.
+	TranscodeToInputPath bool   // Stores the transcodes alongside the raw input
+	PersistTranscodes    bool   // Stores the transcodes in a permanent directory
+	SegmentPrefix        string
 
 	VideoProfile   *VideoProfile
 	VideoKeyframes bool

internal/api/hlsvod.go

@@ -107,6 +107,7 @@ func (a *ApiManagerCtx) HlsVod(r chi.Router) {
 			Str("path", urlPath).
 			Str("hlsResource", hlsResource).
 			Str("vodMediaPath", vodMediaPath).
+			Str("profileID", profileID).
 			Msg("new hls vod request")
 
 		// if manager was not found
@@ -117,19 +118,46 @@ func (a *ApiManagerCtx) HlsVod(r chi.Router) {
 				return
 			}
 
-			// create own transcoding directory
-			transcodeDir, err := os.MkdirTemp(a.config.Vod.TranscodeDir, fmt.Sprintf("vod-%s-*", profileID))
-			if err != nil {
-				logger.Warn().Err(err).Msg("could not create temp dir")
-				http.Error(w, "500 could not create temp dir", http.StatusInternalServerError)
-				return
+			// Check if we want to store transcodes alongside the original media
+			var transcodeSubDir string
+			if a.config.Vod.TranscodeToInputPath {
+				transcodeSubDir = filepath.Dir(vodMediaPath)
+			} else {
+				transcodeSubDir = a.config.Vod.TranscodeDir
+			}
+
+			logger.Info().
+				Bool("persistTranscodes", a.config.Vod.PersistTranscodes).
+				Str("transcodeDir", transcodeSubDir).
+				Msg("creating manager")
+
+			var transcodeDir string
+			// create persistent directory if requested
+			if a.config.Vod.PersistTranscodes {
+				transcodeDir = filepath.Join(transcodeSubDir, fmt.Sprintf("vod-%s", profileID))
+				err := os.Mkdir(transcodeDir, 0750)
+				if err != nil && !os.IsExist(err) {
+					logger.Warn().Err(err).Msg("could not create dir")
+					http.Error(w, "500 could not create dir", http.StatusInternalServerError)
+					return
+				}
+			} else {
+				// create temp transcoding directory
+				var err error
+				transcodeDir, err = os.MkdirTemp(transcodeSubDir, fmt.Sprintf("vod-%s-*", profileID))
+				if err != nil {
+					logger.Warn().Err(err).Msg("could not create temp dir")
+					http.Error(w, "500 could not create temp dir", http.StatusInternalServerError)
+					return
+				}
 			}
 
 			// create new manager
 			manager = hlsvod.New(hlsvod.Config{
-				MediaPath:     vodMediaPath,
-				TranscodeDir:  transcodeDir,
-				SegmentPrefix: profileID,
+				MediaPath:         vodMediaPath,
+				TranscodeDir:      transcodeDir,
+				PersistTranscodes: a.config.Vod.PersistTranscodes,
+				SegmentPrefix:     profileID,
 
 				VideoProfile: &hlsvod.VideoProfile{
 					Width:   profile.Width,