Skip to content

Commit

Permalink
Audio for Videos (#1490)
Browse files Browse the repository at this point in the history 
Co-authored-by: Sjmarf <[email protected]>
  • Loading branch information
@EricBAndrews @Sjmarf
EricBAndrews and Sjmarf authored Dec 11, 2024
1 parent 3daecd4 commit 59ce59d
Show file tree
Hide file tree
Showing 14 changed files with 225 additions and 34 deletions.
12 changes: 12 additions & 0 deletions Mlem.xcodeproj/project.pbxproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -331,6 +331,8 @@
CD4ED8472BF110FA00EFA0A2 /* TabReselectTracker.swift in Sources */ = {isa = PBXBuildFile; fileRef = CD4ED8462BF110FA00EFA0A2 /* TabReselectTracker.swift */; };
CD4ED84A2BF1113800EFA0A2 /* View+TabReselectConsumer.swift in Sources */ = {isa = PBXBuildFile; fileRef = CD4ED8492BF1113800EFA0A2 /* View+TabReselectConsumer.swift */; };
CD5581DE2C7B8B820043FAC3 /* ImageFunctions.swift in Sources */ = {isa = PBXBuildFile; fileRef = CD5581DD2C7B8B820043FAC3 /* ImageFunctions.swift */; };
CD57AFA32D03761500AB3956 /* WebpView.swift in Sources */ = {isa = PBXBuildFile; fileRef = CD57AFA22D03761300AB3956 /* WebpView.swift */; };
CD57AFA52D0377EB00AB3956 /* AnimationControlLayer.swift in Sources */ = {isa = PBXBuildFile; fileRef = CD57AFA42D0377E400AB3956 /* AnimationControlLayer.swift */; };
CD5F3F452D06167F008C55D5 /* MlemMiddleware in Frameworks */ = {isa = PBXBuildFile; productRef = CD5F3F442D06167F008C55D5 /* MlemMiddleware */; };
CD635E1B2C94DACD00864F75 /* BypassProxyWarningSheet.swift in Sources */ = {isa = PBXBuildFile; fileRef = CD635E1A2C94DACD00864F75 /* BypassProxyWarningSheet.swift */; };
CD64A91A2CA37E8D007CA7E6 /* Gifu in Frameworks */ = {isa = PBXBuildFile; productRef = CD64A9192CA37E8D007CA7E6 /* Gifu */; };
Expand All @@ -347,6 +349,7 @@
CD7DB9712C49C17200DCC542 /* PersonContentGridView.swift in Sources */ = {isa = PBXBuildFile; fileRef = CD7DB9702C49C17200DCC542 /* PersonContentGridView.swift */; };
CD7DB9732C4AEDDE00DCC542 /* TileCommentView.swift in Sources */ = {isa = PBXBuildFile; fileRef = CD7DB9722C4AEDDE00DCC542 /* TileCommentView.swift */; };
CD7DB9762C4D6C0A00DCC542 /* FeedCommentView.swift in Sources */ = {isa = PBXBuildFile; fileRef = CD7DB9752C4D6C0A00DCC542 /* FeedCommentView.swift */; };
CD8457142D07576A00CEA2B8 /* AVPlayer+Extensions.swift in Sources */ = {isa = PBXBuildFile; fileRef = CD8457132D07576700CEA2B8 /* AVPlayer+Extensions.swift */; };
CD869FCC2C15F8AC00FC8B5B /* BubblePickerView.swift in Sources */ = {isa = PBXBuildFile; fileRef = CD869FCB2C15F8AC00FC8B5B /* BubblePickerView.swift */; };
CD869FCE2C15F90C00FC8B5B /* ChildSizeReader.swift in Sources */ = {isa = PBXBuildFile; fileRef = CD869FCD2C15F90C00FC8B5B /* ChildSizeReader.swift */; };
CD8DCAF92C5E92E8003E4DD7 /* Profile1Providing+Extensions.swift in Sources */ = {isa = PBXBuildFile; fileRef = CD8DCAF82C5E92E8003E4DD7 /* Profile1Providing+Extensions.swift */; };
Expand Down Expand Up @@ -747,6 +750,8 @@
CD4ED8462BF110FA00EFA0A2 /* TabReselectTracker.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TabReselectTracker.swift; sourceTree = "<group>"; };
CD4ED8492BF1113800EFA0A2 /* View+TabReselectConsumer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "View+TabReselectConsumer.swift"; sourceTree = "<group>"; };
CD5581DD2C7B8B820043FAC3 /* ImageFunctions.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ImageFunctions.swift; sourceTree = "<group>"; };
CD57AFA22D03761300AB3956 /* WebpView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WebpView.swift; sourceTree = "<group>"; };
CD57AFA42D0377E400AB3956 /* AnimationControlLayer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AnimationControlLayer.swift; sourceTree = "<group>"; };
CD635E1A2C94DACD00864F75 /* BypassProxyWarningSheet.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BypassProxyWarningSheet.swift; sourceTree = "<group>"; };
CD64A91B2CA38DA7007CA7E6 /* NukeWebpBridgeDecoder.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NukeWebpBridgeDecoder.swift; sourceTree = "<group>"; };
CD64A91D2CA6255A007CA7E6 /* MediaView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MediaView.swift; sourceTree = "<group>"; };
Expand All @@ -761,6 +766,7 @@
CD7DB9702C49C17200DCC542 /* PersonContentGridView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PersonContentGridView.swift; sourceTree = "<group>"; };
CD7DB9722C4AEDDE00DCC542 /* TileCommentView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TileCommentView.swift; sourceTree = "<group>"; };
CD7DB9752C4D6C0A00DCC542 /* FeedCommentView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FeedCommentView.swift; sourceTree = "<group>"; };
CD8457132D07576700CEA2B8 /* AVPlayer+Extensions.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AVPlayer+Extensions.swift"; sourceTree = "<group>"; };
CD869FCB2C15F8AC00FC8B5B /* BubblePickerView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BubblePickerView.swift; sourceTree = "<group>"; };
CD869FCD2C15F90C00FC8B5B /* ChildSizeReader.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ChildSizeReader.swift; sourceTree = "<group>"; };
CD869FCF2C15F92E00FC8B5B /* Int+Extensions.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Int+Extensions.swift"; sourceTree = "<group>"; };
Expand Down Expand Up @@ -1341,6 +1347,7 @@
CD13CC662C5D3CA7001AF428 /* Helpers */ = {
isa = PBXGroup;
children = (
CD57AFA42D0377E400AB3956 /* AnimationControlLayer.swift */,
CD332D782CA7175200A53988 /* PlayButton.swift */,
CD64A91B2CA38DA7007CA7E6 /* NukeWebpBridgeDecoder.swift */,
CD13CC602C5D262E001AF428 /* MediaLoadingState.swift */,
Expand Down Expand Up @@ -1409,6 +1416,7 @@
CD332D7A2CA71E5D00A53988 /* Animated */ = {
isa = PBXGroup;
children = (
CD57AFA22D03761300AB3956 /* WebpView.swift */,
CD332D7B2CA71E6E00A53988 /* GifView.swift */,
CDE4AC3E2CA2082F00981010 /* VideoView.swift */,
);
Expand Down Expand Up @@ -1558,6 +1566,7 @@
CD4D58C22B86DC5800B82964 /* Extensions */ = {
isa = PBXGroup;
children = (
CD8457132D07576700CEA2B8 /* AVPlayer+Extensions.swift */,
CD332D7D2CA7485D00A53988 /* String+Extensions.swift */,
0397D47F2C693A88002C6CDC /* [BlockNode]+Extensions.swift */,
03A82FA02C0D1E8500D01A5C /* ApiClient+Extensions.swift */,
Expand Down Expand Up @@ -2317,6 +2326,7 @@
CDCA44B42C176A4700C092B3 /* Array+Extensions.swift in Sources */,
03CCDAA42BF2852E00C0C851 /* LoginTotpView.swift in Sources */,
CD4ED8472BF110FA00EFA0A2 /* TabReselectTracker.swift in Sources */,
CD8457142D07576A00CEA2B8 /* AVPlayer+Extensions.swift in Sources */,
033FCAEE2C57DD14007B7CD1 /* CaptchaDifficulty+Extensions.swift in Sources */,
035BE0912BDEA01E00F77D73 /* View+NavigationSheetModifiers.swift in Sources */,
CDAA02DB2C810DB200D75633 /* Calendar+Extensions.swift in Sources */,
Expand All @@ -2334,6 +2344,7 @@
CD8DCAF92C5E92E8003E4DD7 /* Profile1Providing+Extensions.swift in Sources */,
CDB2EC8A2BFAEFDF00DBC0EF /* ThumbnailImageView.swift in Sources */,
0397D4932C6CE87E002C6CDC /* PostEditorTargetView.swift in Sources */,
CD57AFA32D03761500AB3956 /* WebpView.swift in Sources */,
0369B3562BFA6824001EFEDF /* InboxView.swift in Sources */,
CD4D59162B87B38C00B82964 /* UIApplication+Extensions.swift in Sources */,
CDB41E8A2C83C24400BD2DE9 /* Section.swift in Sources */,
Expand Down Expand Up @@ -2385,6 +2396,7 @@
CDC199EA2BE449790077B4F1 /* Interactable1Providing+Extensions.swift in Sources */,
03500C2B2BF7F1B100CAA076 /* ToastOverlayView.swift in Sources */,
032C32042C3439C600595286 /* ActorIdentifiable+Extensions.swift in Sources */,
CD57AFA52D0377EB00AB3956 /* AnimationControlLayer.swift in Sources */,
CD4D59182B87B3B000B82964 /* UIViewController+Extensions.swift in Sources */,
CD13CC572C582DD8001AF428 /* DynamicMediaView.swift in Sources */,
034B94812C09306D00039AF4 /* CommunityOrPersonStub+Extensions.swift in Sources */,
Expand Down
6 changes: 5 additions & 1 deletion Mlem/App/Configuration/Icons.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -282,6 +282,11 @@ enum Icons {
static let fediseerHesitation: String = "exclamationmark.triangle.fill"
static let fediseerCensure: String = "exclamationmark.octagon.fill"

// media
static let play: String = "play.fill"
static let muted: String = "speaker.slash.fill"
static let unmuted: String = "speaker.wave.2.fill"

// misc
static let `private`: String = "lock"
static let email: String = "envelope"
Expand All @@ -307,7 +312,6 @@ enum Icons {
static let imageDetails: String = "doc.badge.ellipsis"
static let accountSwitchReload: String = "arrow.2.circlepath"
static let accountSwitchKeepPlace: String = "checkmark.diamond"
static let play: String = "play.fill"
}

// swiftlint:enable type_body_length
3 changes: 3 additions & 0 deletions Mlem/App/Configuration/User Settings/CodableSettings.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ struct CodableSettings: Codable {
var behavior_internetSpeed: InternetSpeed
var behavior_upvoteOnSave: Bool
var behavior_autoplayMedia: Bool
var behavior_muteVideos: Bool
var behavior_infiniteScroll: Bool
var comment_behaviors_collapseChildren: Bool
var comment_compact: Bool
Expand Down Expand Up @@ -104,6 +105,7 @@ struct CodableSettings: Codable {
self.behavior_hapticLevel = try container.decodeIfPresent(HapticPriority.self, forKey: .behavior_hapticLevel) ?? .high
self.behavior_internetSpeed = try container.decodeIfPresent(InternetSpeed.self, forKey: .behavior_internetSpeed) ?? .fast
self.behavior_autoplayMedia = try container.decodeIfPresent(Bool.self, forKey: .behavior_autoplayMedia) ?? false
self.behavior_muteVideos = try container.decodeIfPresent(Bool.self, forKey: .behavior_muteVideos) ?? true
self.behavior_upvoteOnSave = try container.decodeIfPresent(Bool.self, forKey: .behavior_upvoteOnSave) ?? false
self.behavior_infiniteScroll = try container.decodeIfPresent(Bool.self, forKey: .behavior_infiniteScroll) ?? true
self.comment_behaviors_collapseChildren = try container.decodeIfPresent(Bool.self, forKey: .comment_behaviors_collapseChildren) ?? false
Expand Down Expand Up @@ -182,6 +184,7 @@ struct CodableSettings: Codable {
self.behavior_internetSpeed = settings.internetSpeed
self.behavior_upvoteOnSave = settings.upvoteOnSave
self.behavior_autoplayMedia = settings.autoplayMedia
self.behavior_muteVideos = settings.muteVideos
self.behavior_infiniteScroll = settings.infiniteScroll
self.comment_behaviors_collapseChildren = false
self.comment_compact = settings.compactComments
Expand Down
3 changes: 3 additions & 0 deletions Mlem/App/Configuration/User Settings/Settings.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ class Settings: ObservableObject {
@AppStorage("behavior.upvoteOnSave") var upvoteOnSave: Bool = false
@AppStorage("behavior.internetSpeed") var internetSpeed: InternetSpeed = .fast
@AppStorage("behavior.autoplayMedia") var autoplayMedia: Bool = false
@AppStorage("behavior.muteVideos") var muteVideos: Bool = true
@AppStorage("behavior.confirmImageUploads") var confirmImageUploads: Bool = true
@AppStorage("behavior.infiniteScroll") var infiniteScroll: Bool = true

Expand Down Expand Up @@ -114,6 +115,8 @@ class Settings: ObservableObject {
hapticLevel = settings.behavior_hapticLevel
upvoteOnSave = settings.behavior_upvoteOnSave
internetSpeed = settings.behavior_internetSpeed
autoplayMedia = settings.behavior_autoplayMedia
muteVideos = settings.behavior_muteVideos
infiniteScroll = settings.behavior_infiniteScroll
keepPlaceOnAccountSwitch = settings.accounts_keepPlace
accountSort = settings.accounts_sort
Expand Down
15 changes: 15 additions & 0 deletions Mlem/App/Utility/Extensions/AVPlayer+Extensions.swift
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
//
// AVPlayer+Extensions.swift
// Mlem
//
// Created by Eric Andrews on 2024-12-09.
//
// From https://stackoverflow.com/questions/11704322/how-to-check-if-avplayer-has-video-or-just-audio

import AVFoundation

extension AVPlayer {
func isAudioAvailable() async throws -> Bool? {
return try await self.currentItem?.asset.loadTracks(withMediaType: .audio).count != 0
}
}
9 changes: 9 additions & 0 deletions Mlem/App/Views/Root/MlemApp.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import Nuke
import SDWebImageWebPCoder
import SwiftUI
import AVFAudio

/// Root view for the app
@main
Expand All @@ -28,7 +29,15 @@ struct MlemApp: App {
ImageDecoderRegistry.shared.register(NukeWebpBridgeDecoder.init)
SDImageCodersManager.shared.addCoder(SDImageWebPCoder.shared)

// caching
URLCache.shared = Constants.main.urlCache

// set up audio
do {
try AVAudioSession.sharedInstance().setCategory(.playback)
} catch {
handleError(error)
}
}

var body: some Scene {
Expand Down
2 changes: 2 additions & 0 deletions Mlem/App/Views/Root/Tabs/Settings/GeneralSettingsView.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ struct GeneralSettingsView: View {
@Setting(\.markReadOnScroll) var markReadOnScroll
@Setting(\.infiniteScroll) var infiniteScroll
@Setting(\.autoplayMedia) var autoplayMedia
@Setting(\.muteVideos) var muteVideos
@Setting(\.defaultFeed) var defaultFeed
@Setting(\.sidebarVisibleByDefault) var sidebarVisibleByDefault
@Setting(\.confirmImageUploads) var confirmImageUploads
Expand Down Expand Up @@ -65,6 +66,7 @@ struct GeneralSettingsView: View {
if #available(iOS 18.0, *) {
Toggle("Autoplay Media", isOn: $autoplayMedia)
}
Toggle("Mute Videos", isOn: $muteVideos)
Toggle("Mark Read on Scroll", isOn: $markReadOnScroll)
Toggle("Infinite Scroll", isOn: $infiniteScroll)
Toggle("Upvote on Save", isOn: $upvoteOnSave)
Expand Down
19 changes: 11 additions & 8 deletions Mlem/App/Views/Shared/Images/Core/Animated/GifView.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,28 +10,31 @@ import SwiftUI

struct GifView: View {
let data: Data
@State var animating: Bool = true

var body: some View {
UIGifView(data: data)
.allowsHitTesting(false)
.overlay {
Color.clear.contentShape(.rect)
}
UIGifView(data: data, animating: animating)
.withAnimationControls(animating: $animating)
}
}

private struct UIGifView: UIViewRepresentable {
let data: Data
let animating: Bool

func makeUIView(context: Context) -> some UIView {
func makeUIView(context: Context) -> GIFImageView {
let imageView = GIFImageView()
imageView.setContentCompressionResistancePriority(.defaultLow, for: .horizontal)
imageView.setContentCompressionResistancePriority(.defaultLow, for: .vertical)
imageView.animate(withGIFData: data, loopCount: 0)
return imageView
}

func updateUIView(_ uiView: UIViewType, context: Context) {
// noop
func updateUIView(_ uiView: GIFImageView, context: Context) {
if animating {
uiView.startAnimatingGIF()
} else {
uiView.stopAnimatingGIF()
}
}
}
81 changes: 71 additions & 10 deletions Mlem/App/Views/Shared/Images/Core/Animated/VideoView.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,19 +8,80 @@
import AVFoundation
import NukeVideo
import SwiftUI
import AVKit

struct VideoView: UIViewRepresentable {
let asset: AVAsset
struct VideoView: View {
let player: AVQueuePlayer
let playerLooper: AVPlayerLooper

func makeUIView(context: Context) -> some UIView {
let view = VideoPlayerView()
view.asset = asset
view.videoGravity = .resizeAspect
view.play()
return view
/// Controls the video's animation (true for playing). Defaults to false; video is automatically started once audio is resolved
@State var animating: Bool = false

/// Controls the video's audio state (true for muted).
@State var muted: Bool

/// Whether the video has an audio track. Set post-appearance since this is asynchronously computed.
@State var audioAvailable: Bool = false

/// Observer to track external modifications to the `isMuted` status of the player.
@State var observer: NSKeyValueObservation?

/// Whether this is the first time this view has appeared
@State var isFirstAppearance: Bool = true

init(asset: AVAsset) {
// set up AVQueuePlayer and AVPlayerLooper to loop the video
let playerItem: AVPlayerItem = .init(asset: asset)
player = .init(playerItem: playerItem)
playerLooper = .init(player: player, templateItem: playerItem)

@Setting(\.muteVideos) var muteVideos
player.isMuted = muteVideos
self._muted = .init(wrappedValue: muteVideos)
}

func updateUIView(_ uiView: UIViewType, context: Context) {
// noop
var body: some View {
VideoPlayer(player: player)
.disabled(true)
.onDisappear {
observer = nil
}
.onAppear {
guard observer == nil else { return }

// audio is automatically turned on if the user modifies their volume. This listens to that event and updates muted to match.
observer = player.observe(\.isMuted, options: [.new]) { _, value in
if let newValue = value.newValue, newValue != muted {
muted = newValue
}
}
}
.task {
// parse whether the video has audio or not before playing so we can appropriately display audio controls
do {
audioAvailable = try await player.isAudioAvailable() ?? false
} catch {
handleError(error)
}

// if parse fails, assume no audio and play anyway
if isFirstAppearance {
animating = true
isFirstAppearance = false
}
}
.onChange(of: animating, initial: false) {
if animating {
player.play()
} else {
player.pause()
}
}
.onChange(of: muted, initial: false) {
if player.isMuted != muted {
player.isMuted = muted
}
}
.withAnimationControls(animating: $animating, muted: audioAvailable ? $muted : nil)
}
}
21 changes: 21 additions & 0 deletions Mlem/App/Views/Shared/Images/Core/Animated/WebpView.swift
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
//
// WebpView.swift
// Mlem
//
// Created by Eric Andrews on 2024-12-06.
//

import SDWebImageSwiftUI
import SwiftUI

struct WebpView: View {
let data: Data

@State var animating: Bool = true

var body: some View {
AnimatedImage(data: data, isAnimating: $animating)
.resizable()
.withAnimationControls(animating: $animating)
}
}
Loading

0 comments on commit 59ce59d

Please sign in to comment.