Add Voice Support

AudioSource.swift

@@ -0,0 +1,208 @@
+//
+//  AudioSource.swift
+//  
+//
+//  Created by Noah Pistilli on 2022-05-10.
+//
+
+import Foundation
+import NIO
+import SwiftFFmpeg
+
+/// Overarching protocol that all playable types of audio must inherit.
+public protocol AudioSource {
+
+    /// Reads 20 milliseconds worth of audio.
+    /// If the audio is complete, the subclass should return an empty `Data` object.
+    /// If `AudioSource.isOpus` is `true`, then then it must return
+    /// 20ms worth of Opus encoded audio. Otherwise, it must be 20ms
+    /// worth of 16-bit 48KHz stereo PCM, which is about 3,840 bytes
+    /// per frame (20ms worth of audio).
+    func read() -> Data
+
+    /// Whether or not the current audio source is already encoded in opus.
+    var isOpus: Bool { get }
+}
+
+/// Class that reads raw PCM s16le audio.
+/// PCM must have a sampling rate of 48k and 2 channels.
+public class PCMAudio: AudioSource {
+
+    /// The audio stream in it's entirety
+    var stream: ByteBuffer
+
+    public init(stream: Data) {
+        self.stream = ByteBuffer(data: stream)
+    }
+
+    public func read() -> Data {
+        let ret = self.stream.readData(length: 3840)
+
+        if let ret = ret {
+            return ret
+        }
+
+        return Data()
+    }
+
+    public var isOpus: Bool { false }
+}
+
+/// Class that converts inputed audio source into PCM playable by Discord using FFmpeg.
+public class FFmpegAudio: AudioSource {
+
+    /// The audio stream in it's entirety
+    var stream: ByteBuffer
+
+    // I apologize for the below code. Simply, this function takes any audio format that
+    // FFmpeg can demux and filter it into the PCM s16le Discord wants.
+    public init(input: String) throws {
+        var tempBuffer = Data()
+
+        let fmtCtx = try AVFormatContext(url: input)
+        try fmtCtx.findStreamInfo()
+
+        // Find the type of audio stream
+        let streamIndex = fmtCtx.findBestStream(type: .audio)!
+        let stream = fmtCtx.streams[streamIndex]
+
+        // Create decoder
+        let decoder = AVCodec.findDecoderById(stream.codecParameters.codecId)!
+        let decoderCtx = AVCodecContext(codec: decoder)
+        decoderCtx.setParameters(stream.codecParameters)
+        try decoderCtx.openCodec()
+
+        let buffersrc = AVFilter(name: "abuffer")!
+        let buffersink = AVFilter(name: "abuffersink")!
+        let inputs = AVFilterInOut()
+        let outputs = AVFilterInOut()
+        let sampleFmts = [AVSampleFormat.int16]
+        let channelLayouts = [AVChannelLayout.CHL_STEREO]
+        let sampleRates = [48000] as [CInt]
+        let filterGraph = AVFilterGraph()
+
+        // buffer audio source: the decoded frames from the decoder will be inserted here.
+        let args = """
+          time_base=\(stream.timebase.num)/\(stream.timebase.den):\
+          sample_rate=\(decoderCtx.sampleRate):\
+          sample_fmt=\(decoderCtx.sampleFormat.name!):\
+          channel_layout=0x\(decoderCtx.channelLayout.rawValue)
+          """
+        let buffersrcCtx = try filterGraph.addFilter(buffersrc, name: "in", args: args)
+
+        // buffer audio sink: to terminate the filter chain.
+        let buffersinkCtx = try filterGraph.addFilter(buffersink, name: "out", args: nil)
+        try buffersinkCtx.set(sampleFmts.map({ $0.rawValue }), forKey: "sample_fmts")
+        try buffersinkCtx.set(channelLayouts.map({ $0.rawValue }), forKey: "channel_layouts")
+        try buffersinkCtx.set(sampleRates, forKey: "sample_rates")
+
+        // Set the endpoints for the filter graph.
+        outputs.name = "in"
+        outputs.filterContext = buffersrcCtx
+        outputs.padIndex = 0
+        outputs.next = nil
+
+        // The buffer sink input must be connected to the output pad of
+        // the last filter described by filters_descr; since the last
+        // filter output label is not specified, it is set to "out" by default.
+        inputs.name = "out"
+        inputs.filterContext = buffersinkCtx
+        inputs.padIndex = 0
+        inputs.next = nil
+
+        try filterGraph.parse(
+          filters: "aresample=48000,aformat=sample_fmts=s16:channel_layouts=stereo", inputs: inputs,
+          outputs: outputs)
+        try filterGraph.configure()
+
+        let pkt = AVPacket()
+        let frame = AVFrame()
+        let filterFrame = AVFrame()
+
+        // Read all packets
+        while let _ = try? fmtCtx.readFrame(into: pkt) {
+            defer { pkt.unref() }
+
+            if pkt.streamIndex != streamIndex {
+                continue
+            }
+
+            try decoderCtx.sendPacket(pkt)
+
+            while true {
+                do {
+                    try decoderCtx.receiveFrame(frame)
+                } catch let err as AVError where err == .tryAgain || err == .eof {
+                    break
+                }
+
+                // push the audio data from decoded frame into the filtergraph
+                try buffersrcCtx.addFrame(frame, flags: .keepReference)
+
+                // pull filtered audio from the filtergraph
+                while true {
+                    do {
+                        try buffersinkCtx.getFrame(filterFrame)
+                    } catch let err as AVError where err == .tryAgain || err == .eof {
+                        break
+                    }
+
+                    // Now actually write the data
+                    let n = filterFrame.sampleCount * filterFrame.channelLayout.channelCount
+                    let data = UnsafeRawPointer(filterFrame.data[0]!).bindMemory(to: UInt16.self, capacity: n)
+
+                    for i in 0..<n {
+                        tempBuffer.append(UInt8(data[i] & 0xff))
+                        tempBuffer.append(UInt8(data[i] >> 8 & 0xff))
+                    }
+
+                    filterFrame.unref()
+                }
+                frame.unref()
+            }
+        }
+
+        self.stream = ByteBuffer(data: tempBuffer)
+    }
+
+    public func read() -> Data {
+        let ret = self.stream.readData(length: 3840)
+
+        if let ret = ret {
+            return ret
+        }
+
+        return Data()
+    }
+
+    public var isOpus: Bool { false }
+}
+
+/// Class that plays Opus encoded audio.
+public class OpusAudio: AudioSource {
+
+    /// The audio stream in it's entirety
+    var stream: ByteBuffer
+
+    var opusStream: [[UInt8]]
+
+    var packetPos = -1
+
+    public init(stream: Data) {
+        self.stream = ByteBuffer(data: stream)
+        let packets = OggStream(stream: self.stream).iterPackets()
+        self.opusStream = packets
+    }
+
+    public func read() -> Data {
+        packetPos += 1
+
+        if packetPos < self.opusStream.count {
+            return Data(self.opusStream[packetPos])
+        } else {
+            return Data()
+        }
+    }
+
+    public var isOpus: Bool { true }
+}

Package.resolved

@@ -1,6 +1,15 @@
 {
   "object": {
     "pins": [
+      {
+        "package": "COPUS",
+        "repositoryURL": "https://github.com/nuclearace/copus",
+        "state": {
+          "branch": null,
+          "revision": "365743902efc1c93730757cea288bef4b90637a0",
+          "version": "2.0.0"
+        }
+      },
       {
         "package": "MimeType",
         "repositoryURL": "https://github.com/SketchMaster2001/MimeType.git",
@@ -10,39 +19,66 @@
           "version": null
         }
       },
+      {
+        "package": "Opus",
+        "repositoryURL": "https://github.com/SketchMaster2001/Opus",
+        "state": {
+          "branch": "master",
+          "revision": "7bb0718f8ca6105f427bc2f698b0aaaff8a5016a",
+          "version": null
+        }
+      },
+      {
+        "package": "Sodium",
+        "repositoryURL": "https://github.com/nuclearace/Sodium",
+        "state": {
+          "branch": "master",
+          "revision": "5812a3d879b77aae0fdfbd62d0e8354e914d15ae",
+          "version": null
+        }
+      },
       {
         "package": "swift-log",
         "repositoryURL": "https://github.com/apple/swift-log.git",
         "state": {
-          "branch": null,
-          "revision": "5d66f7ba25daf4f94100e7022febf3c75e37a6c7",
-          "version": "1.4.2"
+          "branch": "main",
+          "revision": "2bf6215b68bbb4a668a47468bcc8de56f00dbc3e",
+          "version": null
         }
       },
       {
         "package": "swift-nio",
         "repositoryURL": "https://github.com/apple/swift-nio.git",
         "state": {
           "branch": null,
-          "revision": "51c3fc2e4a0fcdf4a25089b288dd65b73df1b0ef",
-          "version": "2.37.0"
+          "revision": "124119f0bb12384cef35aa041d7c3a686108722d",
+          "version": "2.40.0"
         }
       },
       {
         "package": "swift-nio-ssl",
         "repositoryURL": "https://github.com/apple/swift-nio-ssl.git",
         "state": {
           "branch": null,
-          "revision": "52a486ff6de9bc3e26bf634c5413c41c5fa89ca5",
-          "version": "2.17.2"
+          "revision": "42436a25ff32c390465567f5c089a9a8ce8d7baf",
+          "version": "2.20.0"
+        }
+      },
+      {
+        "package": "SwiftFFmpeg",
+        "repositoryURL": "https://github.com/sunlubo/SwiftFFmpeg.git",
+        "state": {
+          "branch": "master",
+          "revision": "fdf975bd93513b6007acde46cda9357a52c8f427",
+          "version": null
         }
       },
       {
         "package": "websocket-kit",
         "repositoryURL": "https://github.com/vapor/websocket-kit",
         "state": {
           "branch": "main",
-          "revision": "ff8fbce837ef01a93d49c6fb49a72be0f150dac7",
+          "revision": "09212f4c2b9ebdef00f04b913b57f5d77bc4ea62",
           "version": null
         }
       }

Package.swift

@@ -10,15 +10,33 @@ let package = Package(
   dependencies: [
     // WebSockets for Linux and macOS
     .package(url: "https://github.com/vapor/websocket-kit", .branch("main")),
+
     // Logging for Swift
-    .package(url: "https://github.com/apple/swift-log.git", from: "1.4.2"),
+    .package(url: "https://github.com/apple/swift-log.git", .branch("main")),
+
     // Library that contains common mimetypes
-    .package(url: "https://github.com/SketchMaster2001/MimeType.git", .branch("master"))
+    .package(url: "https://github.com/SketchMaster2001/MimeType.git", .branch("master")),
+
+    // Voice packet encryption
+    .package(url: "https://github.com/nuclearace/Sodium", .branch("master")),
+
+    // Opus bindings for SPM
+    .package(url: "https://github.com/SketchMaster2001/Opus", .branch("master")),
+
+    // FFmpeg bindings for SPM
+    .package(url: "https://github.com/sunlubo/SwiftFFmpeg", branch: "master")
   ],
   targets: [
     .target(
       name: "Swiftcord",
-      dependencies: [.product(name: "WebSocketKit", package: "websocket-kit"), .product(name: "Logging", package: "swift-log"), "MimeType"]
+      dependencies: [
+        .product(name: "WebSocketKit", package: "websocket-kit"),
+        .product(name: "Logging", package: "swift-log"),
+        "MimeType",
+        "Sodium",
+        "Opus",
+        "SwiftFFmpeg"
+      ]
     ),
     .testTarget(
         name: "SwiftcordTests",