From b4ea58baee84b1a55c5126b0ebd8e6285523b975 Mon Sep 17 00:00:00 2001 From: Will Toher Date: Mon, 18 Dec 2023 19:29:55 -0800 Subject: [PATCH 1/2] Add option to send transcriptions to a non-discord webhook --- Cargo.lock | 5 ++ scripty_audio_handler/Cargo.toml | 2 + scripty_audio_handler/src/audio_handler.rs | 68 ++++++++++++++++++- scripty_audio_handler/src/connect.rs | 6 +- .../src/events/client_disconnect.rs | 9 ++- .../src/events/driver_disconnect.rs | 7 +- .../src/events/voice_tick.rs | 8 +-- .../background_tasks/tasks/bot_list_poster.rs | 8 +-- .../src/handler/normal/voice_state_update.rs | 1 + scripty_commands/Cargo.toml | 1 + scripty_commands/src/cmds/join.rs | 28 ++++++-- scripty_core/Cargo.toml | 1 + scripty_core/src/lib.rs | 2 + scripty_i18n/locales/en.ftl | 2 + scripty_utils/Cargo.toml | 1 + scripty_utils/src/http.rs | 16 +++++ scripty_utils/src/lib.rs | 3 + 17 files changed, 146 insertions(+), 22 deletions(-) create mode 100644 scripty_utils/src/http.rs diff --git a/Cargo.lock b/Cargo.lock index 6b0b3238..a1a2ed0d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2724,6 +2724,7 @@ dependencies = [ "backtrace", "dashmap", "parking_lot", + "reqwest", "scripty_automod", "scripty_data_storage", "scripty_db", @@ -2737,6 +2738,7 @@ dependencies = [ "sqlx", "tokio", "tracing", + "url", ] [[package]] @@ -2836,6 +2838,7 @@ dependencies = [ "tokio", "tracing", "typesize", + "url", ] [[package]] @@ -2864,6 +2867,7 @@ dependencies = [ "scripty_metrics", "scripty_redis", "scripty_stt", + "scripty_utils", "scripty_webserver", "tokio", "tracing", @@ -3011,6 +3015,7 @@ dependencies = [ "num", "num_cpus", "once_cell", + "reqwest", "scripty_config", "scripty_db", "serenity", diff --git a/scripty_audio_handler/Cargo.toml b/scripty_audio_handler/Cargo.toml index c88b86b6..b4948914 100644 --- a/scripty_audio_handler/Cargo.toml +++ b/scripty_audio_handler/Cargo.toml @@ -13,6 +13,7 @@ tracing = "0.1" backtrace = "0.3" async-trait = "0.1" parking_lot = "0.12" +reqwest = { version = "0.11", default-features = false, features = ["json", "rustls"] } scripty_db = { path = "../scripty_db" } scripty_stt = { path = "../scripty_stt" } #scripty_tts = { path = "../scripty_tts" } @@ -41,3 +42,4 @@ serenity = { git = "https://github.com/serenity-rs/serenity", branch = "next", f "utils", ] } sqlx = { version = "0.7", features = ["postgres", "macros", "migrate", "runtime-tokio-rustls"] } +url = "2" diff --git a/scripty_audio_handler/src/audio_handler.rs b/scripty_audio_handler/src/audio_handler.rs index 885edec9..55472e4d 100644 --- a/scripty_audio_handler/src/audio_handler.rs +++ b/scripty_audio_handler/src/audio_handler.rs @@ -10,15 +10,19 @@ use ahash::RandomState; use dashmap::{DashMap, DashSet}; use parking_lot::RwLock; use scripty_automod::types::AutomodServerConfig; +use scripty_utils::get_thirdparty_http; use serenity::{ all::RoleId, + builder::ExecuteWebhook, client::Context, + http::CacheHttp, model::{ id::{ChannelId, GuildId}, - webhook::Webhook, + webhook::Webhook as SerenityWebhook, }, }; use songbird::{Event, EventContext, EventHandler}; +use url::Url; use crate::{ events::*, @@ -55,7 +59,7 @@ pub struct AudioHandler { channel_id: ChannelId, voice_channel_id: ChannelId, thread_id: Option, - webhook: Arc, + webhook: Arc, context: Context, premium_level: Arc, verbose: Arc, @@ -68,10 +72,68 @@ pub struct AudioHandler { translate: Arc, } +#[derive(Clone)] +pub struct WebhookWrapper { + discord_webhook: SerenityWebhook, + url_override: Option, +} + +impl WebhookWrapper { + pub fn new(discord_webhook: SerenityWebhook, url_override: Option) -> WebhookWrapper { + WebhookWrapper { + discord_webhook, + url_override, + } + } + + pub async fn execute( + &self, + cache_http: impl CacheHttp, + wait: bool, + builder: ExecuteWebhook, + ) -> Result<(), serenity::Error> { + match &self.url_override { + Some(url) => { + match get_thirdparty_http() + .post(url.clone()) + .json(&builder) + .send() + .await + { + Err(e) => Err(serenity::Error::Http( + serenity::prelude::HttpError::Request(e), + )), + Ok(_) => { + // TODO: debug logging + Ok(()) + } + } + } + None => { + match self + .discord_webhook + .execute(cache_http, wait, builder) + .await + { + Err(e) => Err(e), + Ok(_) => { + // TODO: debug logging + Ok(()) + } + } + } + } + } + + pub fn get_url_override(&self) -> Option { + self.url_override.clone() + } +} + impl AudioHandler { pub async fn new( guild_id: GuildId, - webhook: Webhook, + webhook: WebhookWrapper, context: Context, channel_id: ChannelId, voice_channel_id: ChannelId, diff --git a/scripty_audio_handler/src/connect.rs b/scripty_audio_handler/src/connect.rs index 7e169141..ab813dd1 100644 --- a/scripty_audio_handler/src/connect.rs +++ b/scripty_audio_handler/src/connect.rs @@ -8,7 +8,7 @@ use serenity::{ }; use songbird::{error::JoinError, events::Event, CoreEvent}; -use crate::Error; +use crate::{audio_handler::WebhookWrapper, Error}; // TODO: implement `force` #[allow(clippy::let_unit_value)] @@ -20,12 +20,13 @@ pub async fn connect_to_vc( thread_id: Option, _force: bool, record_transcriptions: bool, + webhook_override_url: Option, ) -> Result<(), Error> { debug!(%guild_id, "fetching webhook"); // thanks to Discord undocumented breaking changes, we have to do this // <3 shitcord let hooks = channel_id.webhooks(&ctx).await?; - let webhook = if hooks.is_empty() { + let discord_webhook = if hooks.is_empty() { channel_id .create_webhook(&ctx, CreateWebhook::new("Scripty Transcriptions")) .await? @@ -48,6 +49,7 @@ pub async fn connect_to_vc( } } }; + let webhook = WebhookWrapper::new(discord_webhook, webhook_override_url); // automatically leave after the specified time period let premium_tier = scripty_premium::get_guild(guild_id.get()).await; diff --git a/scripty_audio_handler/src/events/client_disconnect.rs b/scripty_audio_handler/src/events/client_disconnect.rs index edf8a865..5fce0b6a 100644 --- a/scripty_audio_handler/src/events/client_disconnect.rs +++ b/scripty_audio_handler/src/events/client_disconnect.rs @@ -4,19 +4,22 @@ use std::sync::{ }; use serenity::{ - all::{ChannelId, Context, Webhook}, + all::{ChannelId, Context}, builder::ExecuteWebhook, }; use songbird::model::payload::ClientDisconnect; -use crate::{audio_handler::ArcSsrcMaps, types::TranscriptResults}; +use crate::{ + audio_handler::{ArcSsrcMaps, WebhookWrapper}, + types::TranscriptResults, +}; pub async fn client_disconnect( client_disconnect_data: ClientDisconnect, ssrc_state: ArcSsrcMaps, premium_level: Arc, ctx: Context, - webhook: Arc, + webhook: Arc, thread_id: Option, transcript_results: TranscriptResults, ) { diff --git a/scripty_audio_handler/src/events/driver_disconnect.rs b/scripty_audio_handler/src/events/driver_disconnect.rs index 05c21861..57f841e4 100644 --- a/scripty_audio_handler/src/events/driver_disconnect.rs +++ b/scripty_audio_handler/src/events/driver_disconnect.rs @@ -4,11 +4,12 @@ use serenity::{ all::UserId, builder::{CreateAttachment, CreateMessage, ExecuteWebhook}, client::Context, - model::{id::ChannelId, webhook::Webhook}, + model::id::ChannelId, }; use songbird::{events::context_data::DisconnectReason, id::GuildId, model::CloseCode}; use crate::{ + audio_handler::WebhookWrapper, connect_to_vc, error::ErrorKind, types::{SeenUsers, TranscriptResults}, @@ -18,7 +19,7 @@ pub async fn driver_disconnect( guild_id: GuildId, reason: Option, ctx: Context, - webhook: Arc, + webhook: Arc, channel_id: ChannelId, voice_channel_id: ChannelId, thread_id: Option, @@ -76,6 +77,7 @@ pub async fn driver_disconnect( // retry connection in 30 seconds let record_transcriptions = transcript_results.is_some(); let webhook2 = webhook.clone(); + let webhook_override_url = webhook.get_url_override(); let ctx2 = ctx.clone(); let ctx3 = ctx.clone(); tokio::spawn(async move { @@ -91,6 +93,7 @@ pub async fn driver_disconnect( thread_id, false, record_transcriptions, + webhook_override_url, ) .await .map_err(|x| x.kind) diff --git a/scripty_audio_handler/src/events/voice_tick.rs b/scripty_audio_handler/src/events/voice_tick.rs index 321e8816..ae835e4b 100644 --- a/scripty_audio_handler/src/events/voice_tick.rs +++ b/scripty_audio_handler/src/events/voice_tick.rs @@ -13,14 +13,14 @@ use scripty_automod::types::{AutomodRuleAction, AutomodServerConfig}; use scripty_metrics::Metrics; use scripty_stt::{ModelError, Stream}; use serenity::{ - all::{ChannelId as SerenityChannelId, ChannelId, GuildId, Webhook}, + all::{ChannelId as SerenityChannelId, ChannelId, GuildId}, builder::{CreateEmbed, CreateMessage, EditMember, ExecuteWebhook}, client::Context, }; use songbird::events::context_data::VoiceTick; use crate::{ - audio_handler::SsrcMaps, + audio_handler::{SsrcMaps, WebhookWrapper}, consts::SIZE_OF_I16, types::{SsrcUserDataMap, TranscriptResults}, }; @@ -32,7 +32,7 @@ pub async fn voice_tick( language: Arc>, verbose: Arc, ctx: Context, - webhook: Arc, + webhook: Arc, thread_id: Option, transcript_results: Option>>>, automod_server_cfg: Arc, @@ -108,7 +108,7 @@ async fn handle_silent_speakers( automod_server_cfg, transcript_results, ctx, - auto_detect_lang, + auto_detect_lang: _, translate, }: SilentSpeakersContext<'_>, ) -> Vec<(ExecuteWebhook, u32)> { diff --git a/scripty_bot_utils/src/background_tasks/tasks/bot_list_poster.rs b/scripty_bot_utils/src/background_tasks/tasks/bot_list_poster.rs index 500e8aab..e02fd345 100644 --- a/scripty_bot_utils/src/background_tasks/tasks/bot_list_poster.rs +++ b/scripty_bot_utils/src/background_tasks/tasks/bot_list_poster.rs @@ -5,12 +5,13 @@ use scripty_botlists::*; use scripty_config::BotListsConfig; use serenity::client::Context; +use scripty_utils::get_thirdparty_http; + use crate::{background_tasks::core::BackgroundTask, Error}; pub struct BotListUpdater { ctx: Context, bot_lists: Arc>, - client: reqwest::Client, } #[async_trait] @@ -18,14 +19,12 @@ impl BackgroundTask for BotListUpdater { async fn init(ctx: Context) -> Result { let mut bot_lists = vec![]; let bot_id = ctx.cache.current_user().id.get(); - let client = reqwest::Client::new(); add_bot_lists(&mut bot_lists, bot_id); Ok(Self { ctx, bot_lists: Arc::new(bot_lists), - client, }) } @@ -38,9 +37,10 @@ impl BackgroundTask for BotListUpdater { server_count: self.ctx.cache.guild_count(), shard_count: self.ctx.cache.shard_count().get(), }; + let client = get_thirdparty_http(); for list in self.bot_lists.iter() { - if let Err(e) = list.post_stats(&self.client, stats).await { + if let Err(e) = list.post_stats(&client, stats).await { error!("Failed to post stats to bot list: {}", e); } } diff --git a/scripty_bot_utils/src/handler/normal/voice_state_update.rs b/scripty_bot_utils/src/handler/normal/voice_state_update.rs index 7e7e6731..2f2eaba1 100644 --- a/scripty_bot_utils/src/handler/normal/voice_state_update.rs +++ b/scripty_bot_utils/src/handler/normal/voice_state_update.rs @@ -152,6 +152,7 @@ pub async fn voice_state_update(ctx: Context, _: Option, new: VoiceS None, false, false, + None, ) .await { diff --git a/scripty_commands/Cargo.toml b/scripty_commands/Cargo.toml index 62da8328..a1e0513c 100644 --- a/scripty_commands/Cargo.toml +++ b/scripty_commands/Cargo.toml @@ -40,3 +40,4 @@ serenity = { git = "https://github.com/serenity-rs/serenity", branch = "next", f sqlx = { version = "0.7", features = ["postgres", "macros", "migrate", "runtime-tokio-rustls", "time"] } poise = { git = "https://github.com/serenity-rs/poise", branch = "serenity-next", features = ["cache", "collector"] } rand = "0.8.5" +url = "2" diff --git a/scripty_commands/src/cmds/join.rs b/scripty_commands/src/cmds/join.rs index e41e813d..873a1264 100644 --- a/scripty_commands/src/cmds/join.rs +++ b/scripty_commands/src/cmds/join.rs @@ -8,6 +8,7 @@ use serenity::{ model::channel::{ChannelType, GuildChannel}, prelude::Mentionable, }; +use url::Url; use crate::{Context, Error}; @@ -40,6 +41,9 @@ pub async fn join( #[description = "Create a new thread for this transcription? Defaults to false."] create_thread: Option, + + #[description = "Send transcribed messages to this webhook instead of Discord."] + message_webhook: Option, ) -> Result<(), Error> { let resolved_language = scripty_i18n::get_resolved_language(ctx.author().id.get(), ctx.guild_id().map(|g| g.get())) @@ -248,10 +252,25 @@ pub async fn join( (None, target_channel.id) }; - let output_channel_mention = if let Some(ref target_thread) = target_thread { - target_thread.mention().to_string() - } else { - target_channel.mention().to_string() + let mut url_override: Option = None; + match message_webhook { + Some(url_str) => match Url::parse(&url_str) { + Ok(parsed_url) => url_override = Some(parsed_url), + Err(e) => { + ctx.say( + format_message!(resolved_language, "join-invalid-url", error: e.to_string()), + ) + .await?; + return Ok(()); + } + }, + None => (), + } + + let output_channel_mention = match (&target_thread, &url_override) { + (_, Some(url)) => url.to_string(), + (Some(thread), _) => thread.mention().to_string(), + (_, _) => target_channel.mention().to_string(), }; let res = scripty_audio_handler::connect_to_vc( ctx.serenity_context().clone(), @@ -261,6 +280,7 @@ pub async fn join( target_thread.map(|x| x.id), false, record_transcriptions, + url_override, ) .await; match res { diff --git a/scripty_core/Cargo.toml b/scripty_core/Cargo.toml index ea65c314..26c8f664 100644 --- a/scripty_core/Cargo.toml +++ b/scripty_core/Cargo.toml @@ -24,3 +24,4 @@ scripty_webserver = { path = "../scripty_webserver" } tokio = { version = "1", features = ["parking_lot", "rt-multi-thread"] } scripty_data_storage = { path = "../scripty_data_storage" } fenrir-rs = { git = "https://github.com/tazz4843/fenrir-rs", branch = "json-logs", features = ["reqwest-async", "json-log-fmt"] } +scripty_utils = { path = "../scripty_utils" } diff --git a/scripty_core/src/lib.rs b/scripty_core/src/lib.rs index 3b4cf7be..a87167ab 100644 --- a/scripty_core/src/lib.rs +++ b/scripty_core/src/lib.rs @@ -22,6 +22,8 @@ pub fn start() { scripty_i18n::init_i18n(); + scripty_utils::init_thirdparty_http(); + rt.block_on(async_init()); rt.spawn(scripty_webserver::entrypoint()); rt.block_on(scripty_bot::entrypoint()); diff --git a/scripty_i18n/locales/en.ftl b/scripty_i18n/locales/en.ftl index dc86fb33..b3cfc146 100644 --- a/scripty_i18n/locales/en.ftl +++ b/scripty_i18n/locales/en.ftl @@ -59,6 +59,8 @@ join-forum-requires-tags = The forum channel you tried to make me use requires t join-target-not-text-based = The channel you told me to send transcripts to ({ $targetMention }) is not a text-based channel. Please use a text-based channel, or pick a different channel in the `target_channel` argument. # This message is shown when the user requests the bot create a new thread in a channel, but the channel doesn't support threads being created (usually voice channels) join-create-thread-in-unsupported = Discord does not support threads in { $targetMention }. Please use a different channel, or do not create a thread. +# This message is shown when the user provides an invalid webhook url. +join-invalid-url = The url provided is invalid. Error: { $error } ## Leave command # This and all attributes show up exclusively in the slash command picker when `leave` is selected. diff --git a/scripty_utils/Cargo.toml b/scripty_utils/Cargo.toml index fa8d5f73..b09660cf 100644 --- a/scripty_utils/Cargo.toml +++ b/scripty_utils/Cargo.toml @@ -14,6 +14,7 @@ tracing = "0.1" num_cpus = "1" thousands = "0.2" once_cell = "1" +reqwest = { version = "0.11", default-features = false, features = ["json", "rustls"] } systemstat = "0.2" scripty_db = { path = "../scripty_db" } scripty_config = { path = "../scripty_config" } diff --git a/scripty_utils/src/http.rs b/scripty_utils/src/http.rs new file mode 100644 index 00000000..8443201e --- /dev/null +++ b/scripty_utils/src/http.rs @@ -0,0 +1,16 @@ +use once_cell::sync::OnceCell; + +/// Http client for non-Discord requests +static THIRDPARTY_HTTP_CLIENT: OnceCell = OnceCell::new(); + +pub fn get_thirdparty_http() -> &'static reqwest::Client { + THIRDPARTY_HTTP_CLIENT + .get() + .expect("http should be set before calling get_http") +} + +pub fn init_thirdparty_http() { + THIRDPARTY_HTTP_CLIENT + .set(reqwest::Client::new()) + .unwrap_or_else(|_| panic!("init_thirdparty_http should be called only once")) +} diff --git a/scripty_utils/src/lib.rs b/scripty_utils/src/lib.rs index 64e33f02..637751d4 100644 --- a/scripty_utils/src/lib.rs +++ b/scripty_utils/src/lib.rs @@ -5,12 +5,15 @@ use serenity::{gateway::ShardManager, prelude::TypeMapKey}; mod embed_pagination; mod hash_user_id; mod hex_vec; +mod http; pub mod latency; mod separate_num; pub use embed_pagination::do_paginate; pub use hash_user_id::hash_user_id; pub use hex_vec::vec_to_hex; +pub use http::get_thirdparty_http; +pub use http::init_thirdparty_http; pub use separate_num::separate_num; pub struct ShardManagerWrapper; From b4189997fac44628232c609cd48f9b387fd58308 Mon Sep 17 00:00:00 2001 From: Will Toher Date: Mon, 18 Dec 2023 19:44:46 -0800 Subject: [PATCH 2/2] Format --- .../src/background_tasks/tasks/bot_list_poster.rs | 3 +-- scripty_utils/src/lib.rs | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/scripty_bot_utils/src/background_tasks/tasks/bot_list_poster.rs b/scripty_bot_utils/src/background_tasks/tasks/bot_list_poster.rs index e02fd345..03dc9140 100644 --- a/scripty_bot_utils/src/background_tasks/tasks/bot_list_poster.rs +++ b/scripty_bot_utils/src/background_tasks/tasks/bot_list_poster.rs @@ -3,9 +3,8 @@ use std::{sync::Arc, time::Duration}; use reqwest::Client; use scripty_botlists::*; use scripty_config::BotListsConfig; -use serenity::client::Context; - use scripty_utils::get_thirdparty_http; +use serenity::client::Context; use crate::{background_tasks::core::BackgroundTask, Error}; diff --git a/scripty_utils/src/lib.rs b/scripty_utils/src/lib.rs index 637751d4..4dc61281 100644 --- a/scripty_utils/src/lib.rs +++ b/scripty_utils/src/lib.rs @@ -12,8 +12,7 @@ mod separate_num; pub use embed_pagination::do_paginate; pub use hash_user_id::hash_user_id; pub use hex_vec::vec_to_hex; -pub use http::get_thirdparty_http; -pub use http::init_thirdparty_http; +pub use http::{get_thirdparty_http, init_thirdparty_http}; pub use separate_num::separate_num; pub struct ShardManagerWrapper;