diff --git a/pkg/config/output.go b/pkg/config/output.go
index 7f862698..954c7e79 100644
--- a/pkg/config/output.go
+++ b/pkg/config/output.go
@@ -148,6 +148,14 @@ func (p *PipelineConfig) updateEncodedOutputs(req EncodedOutput) error {
 		}
 	}
 
+	if segmentConf := p.Outputs[types.EgressTypeSegments]; segmentConf != nil {
+		// double the segment length
+		p.KeyFrameInterval = float64(2 * segmentConf.(*SegmentConfig).SegmentDuration)
+	} else if p.KeyFrameInterval == 0 && p.Outputs[types.EgressTypeStream] != nil {
+		// default 4s for streams
+		p.KeyFrameInterval = 4
+	}
+
 	if p.OutputCount == 0 {
 		return errors.ErrInvalidInput("output")
 	}
diff --git a/pkg/config/output_segment.go b/pkg/config/output_segment.go
index fb07b52d..ba822d46 100644
--- a/pkg/config/output_segment.go
+++ b/pkg/config/output_segment.go
@@ -64,19 +64,7 @@ func (p *PipelineConfig) getSegmentConfig(segments *livekit.SegmentedFileOutput)
 	}
 
 	if conf.SegmentDuration == 0 {
-		if p.KeyFrameInterval >= 1 {
-			conf.SegmentDuration = int(p.KeyFrameInterval)
-		} else {
-			conf.SegmentDuration = 4
-		}
-	}
-
-	if p.KeyFrameInterval == 0 {
-		// The splitMuxSink should request key frames from the encoder at expected frame boundaries.
-		// Set the key frame interval to twice the segment duration as a failsafe
-		p.KeyFrameInterval = 2 * float64(conf.SegmentDuration)
-	} else if p.KeyFrameInterval < float64(conf.SegmentDuration) && p.KeyFrameInterval >= 1 {
-		conf.SegmentDuration = int(p.KeyFrameInterval)
+		conf.SegmentDuration = 4
 	}
 
 	switch segments.Protocol {
diff --git a/pkg/config/output_stream.go b/pkg/config/output_stream.go
index a32fea05..92940f86 100644
--- a/pkg/config/output_stream.go
+++ b/pkg/config/output_stream.go
@@ -71,10 +71,5 @@ func (p *PipelineConfig) getStreamConfig(outputType types.OutputType, urls []str
 		p.AudioOutCodec = types.MimeTypeRawAudio
 	}
 
-	// Use a 4s default key frame interval for streaming
-	if p.KeyFrameInterval == 0 {
-		p.KeyFrameInterval = 4
-	}
-
 	return conf, nil
 }