From ab9518593dea45e316894624aaae4bc94741ee21 Mon Sep 17 00:00:00 2001 From: "Carson M." Date: Fri, 26 Jan 2024 11:30:29 -0600 Subject: [PATCH 1/9] initial youtube live chat support --- Cargo.toml | 25 ++- README.md | 2 +- examples/{main.rs => twitch.rs} | 6 +- examples/youtube.rs | 12 ++ src/lib.rs | 83 +------- src/{ => twitch}/event.rs | 0 src/{ => twitch}/identity.rs | 0 src/twitch/mod.rs | 79 ++++++++ src/youtube/mod.rs | 323 ++++++++++++++++++++++++++++++++ src/youtube/types.rs | 319 +++++++++++++++++++++++++++++++ src/youtube/util.rs | 29 +++ 11 files changed, 789 insertions(+), 89 deletions(-) rename examples/{main.rs => twitch.rs} (55%) create mode 100644 examples/youtube.rs rename src/{ => twitch}/event.rs (100%) rename src/{ => twitch}/identity.rs (100%) create mode 100644 src/twitch/mod.rs create mode 100644 src/youtube/mod.rs create mode 100644 src/youtube/types.rs create mode 100644 src/youtube/util.rs diff --git a/Cargo.toml b/Cargo.toml index a867f96..d446949 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,24 +1,33 @@ [package] name = "brainrot" +description = "A live chat interface for Twitch & YouTube" version = "0.1.0" +authors = [ "Carson M. " ] +repository = "https://github.com/vitri-ent/brainrot" edition = "2021" -rust-version = "1.64" +rust-version = "1.75" [dependencies] -irc = { version = "0.15", default-features = false } -tokio = { version = "1", features = [ "net" ] } +irc = { version = "0.15", optional = true, default-features = false } +tokio = { version = "1.0", default-features = false, features = [ "net" ] } futures-util = { version = "0.3", default-features = false } thiserror = "1.0" chrono = { version = "0.4", default-features = false, features = [ "clock", "std" ] } serde = { version = "1.0", optional = true, features = [ "derive" ] } -uuid = "1.5" +serde-aux = { version = "4.4", optional = true } +uuid = { version = "1.5", optional = true } +reqwest = { version = "0.11", optional = true } +simd-json = { version = "0.13", optional = true } +regex = { version = "1.10", optional = true } [dev-dependencies] anyhow = "1.0" tokio = { version = "1", features = [ "rt", "rt-multi-thread", "macros", "net" ] } [features] -default = [ "tls-native" ] -serde = [ "dep:serde", "chrono/serde", "uuid/serde" ] -tls-native = [ "irc/tls-native" ] -tls-rust = [ "irc/tls-rust" ] +default = [ "tls-native", "twitch", "youtube" ] +twitch = [ "dep:irc", "dep:uuid" ] +youtube = [ "dep:simd-json", "dep:reqwest", "dep:serde", "dep:regex", "dep:serde-aux" ] +serde = [ "dep:serde", "chrono/serde", "uuid?/serde" ] +tls-native = [ "irc?/tls-native" ] +#tls-rust = [ "irc?/tls-rust" ] diff --git a/README.md b/README.md index c7d9a70..1e27516 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # `brainrot` -A Twitch IRC client. +A live chat interface for Twitch & YouTube written in Rust. ## Usage See [`examples/main.rs`](https://github.com/vitri-ent/brainrot/blob/examples/main.rs). diff --git a/examples/main.rs b/examples/twitch.rs similarity index 55% rename from examples/main.rs rename to examples/twitch.rs index 45fff49..4608d2f 100644 --- a/examples/main.rs +++ b/examples/twitch.rs @@ -1,14 +1,14 @@ use std::env::args; -use brainrot::ChatEvent; +use brainrot::{twitch, TwitchChat, TwitchChatEvent}; use futures_util::StreamExt; #[tokio::main] async fn main() -> anyhow::Result<()> { - let mut client = brainrot::Chat::new(args().nth(1).as_deref().unwrap_or("miyukiwei"), brainrot::Anonymous).await?; + let mut client = TwitchChat::new(args().nth(1).as_deref().unwrap_or("miyukiwei"), twitch::Anonymous).await?; while let Some(message) = client.next().await.transpose()? { - if let ChatEvent::Message { user, contents, .. } = message { + if let TwitchChatEvent::Message { user, contents, .. } = message { println!("{}: {}", user.display_name, contents.iter().map(|c| c.to_string()).collect::()); } } diff --git a/examples/youtube.rs b/examples/youtube.rs new file mode 100644 index 0000000..d558f87 --- /dev/null +++ b/examples/youtube.rs @@ -0,0 +1,12 @@ +use std::env::args; + +use brainrot::youtube; +use futures_util::StreamExt; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + let (options, cont) = youtube::get_options_from_live_page("e-5D_Shoozk").await?; + let initial_chat = youtube::fetch_yt_chat_page(&options, cont).await?; + youtube::subscribe_to_events(&options, &initial_chat).await?; + Ok(()) +} diff --git a/src/lib.rs b/src/lib.rs index 74192ac..081b870 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,80 +1,9 @@ -use std::{ - pin::Pin, - task::{Context, Poll} -}; +#[cfg(feature = "twitch")] +pub mod twitch; +#[cfg(feature = "twitch")] +pub use self::twitch::{Chat as TwitchChat, ChatEvent as TwitchChatEvent, MessageSegment as TwitchMessageSegment, TwitchIdentity}; -use futures_util::{Stream, StreamExt}; -use irc::{ - client::{prelude::Config, Client, ClientStream}, - proto::Capability -}; +#[cfg(feature = "youtube")] +pub mod youtube; -pub mod identity; -pub use self::identity::{Anonymous, Authenticated, TwitchIdentity}; -mod event; -pub use self::event::{ChatEvent, MessageSegment, User, UserRole}; pub(crate) mod util; - -const TWITCH_SECURE_IRC: (&str, u16) = ("irc.chat.twitch.tv", 6697); -const TWITCH_CAPABILITY_TAGS: Capability = Capability::Custom("twitch.tv/tags"); -const TWITCH_CAPABILITY_MEMBERSHIP: Capability = Capability::Custom("twitch.tv/membership"); -const TWITCH_CAPABILITY_COMMANDS: Capability = Capability::Custom("twitch.tv/commands"); - -/// A connection to a Twitch IRC channel. -/// -/// In order for the connection to stay alive, the IRC client must be able to receive and respond to ping messages, thus -/// you must poll the stream for as long as you wish the client to stay alive. If that isn't possible, start a dedicated -/// thread for the client and send chat events back to your application over an `mpsc` or other channel. -#[derive(Debug)] -pub struct Chat { - stream: ClientStream -} - -impl Chat { - /// Connect to a Twitch IRC channel. - /// - /// ```no_run - /// use brainrot::{Anonymous, Chat}; - /// - /// # #[tokio::main] - /// # async fn main() -> anyhow::Result<()> { - /// let mut client = Chat::new("miyukiwei", Anonymous).await?; - /// # Ok(()) - /// # } - /// ``` - pub async fn new(channel: impl AsRef, auth: impl TwitchIdentity) -> irc::error::Result { - let (username, password) = auth.as_identity(); - let mut client = Client::from_config(Config { - server: Some(TWITCH_SECURE_IRC.0.to_string()), - port: Some(TWITCH_SECURE_IRC.1), - nickname: Some(username.to_string()), - password: password.map(|c| format!("oauth:{c}")), - channels: vec![format!("#{}", channel.as_ref())], - ..Default::default() - }) - .await?; - client.send_cap_req(&[TWITCH_CAPABILITY_COMMANDS, TWITCH_CAPABILITY_MEMBERSHIP, TWITCH_CAPABILITY_TAGS])?; - client.identify()?; - Ok(Self { stream: client.stream()? }) - } -} - -impl Stream for Chat { - type Item = irc::error::Result; - - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let next = self.stream.poll_next_unpin(cx); - match next { - Poll::Ready(Some(Ok(r))) => match self::event::to_chat_event(r) { - Some(ev) => Poll::Ready(Some(Ok(ev))), - None => { - cx.waker().wake_by_ref(); - Poll::Pending - } - }, - Poll::Ready(Some(Err(e))) => Poll::Ready(Some(Err(e))), - Poll::Ready(None) => Poll::Ready(None), - Poll::Pending => Poll::Pending - } - } -} diff --git a/src/event.rs b/src/twitch/event.rs similarity index 100% rename from src/event.rs rename to src/twitch/event.rs diff --git a/src/identity.rs b/src/twitch/identity.rs similarity index 100% rename from src/identity.rs rename to src/twitch/identity.rs diff --git a/src/twitch/mod.rs b/src/twitch/mod.rs new file mode 100644 index 0000000..2d90961 --- /dev/null +++ b/src/twitch/mod.rs @@ -0,0 +1,79 @@ +use std::{ + pin::Pin, + task::{Context, Poll} +}; + +use futures_util::{Stream, StreamExt}; +use irc::{ + client::{prelude::Config, Client, ClientStream}, + proto::Capability +}; + +pub mod identity; +pub use self::identity::{Anonymous, Authenticated, TwitchIdentity}; +mod event; +pub use self::event::{ChatEvent, MessageSegment, User, UserRole}; + +const TWITCH_SECURE_IRC: (&str, u16) = ("irc.chat.twitch.tv", 6697); +const TWITCH_CAPABILITY_TAGS: Capability = Capability::Custom("twitch.tv/tags"); +const TWITCH_CAPABILITY_MEMBERSHIP: Capability = Capability::Custom("twitch.tv/membership"); +const TWITCH_CAPABILITY_COMMANDS: Capability = Capability::Custom("twitch.tv/commands"); + +/// A connection to a Twitch IRC channel. +/// +/// In order for the connection to stay alive, the IRC client must be able to receive and respond to ping messages, thus +/// you must poll the stream for as long as you wish the client to stay alive. If that isn't possible, start a dedicated +/// thread for the client and send chat events back to your application over an `mpsc` or other channel. +#[derive(Debug)] +pub struct Chat { + stream: ClientStream +} + +impl Chat { + /// Connect to a Twitch IRC channel. + /// + /// ```no_run + /// use brainrot::{Anonymous, Chat}; + /// + /// # #[tokio::main] + /// # async fn main() -> anyhow::Result<()> { + /// let mut client = Chat::new("miyukiwei", Anonymous).await?; + /// # Ok(()) + /// # } + /// ``` + pub async fn new(channel: impl AsRef, auth: impl TwitchIdentity) -> irc::error::Result { + let (username, password) = auth.as_identity(); + let mut client = Client::from_config(Config { + server: Some(TWITCH_SECURE_IRC.0.to_string()), + port: Some(TWITCH_SECURE_IRC.1), + nickname: Some(username.to_string()), + password: password.map(|c| format!("oauth:{c}")), + channels: vec![format!("#{}", channel.as_ref())], + ..Default::default() + }) + .await?; + client.send_cap_req(&[TWITCH_CAPABILITY_COMMANDS, TWITCH_CAPABILITY_MEMBERSHIP, TWITCH_CAPABILITY_TAGS])?; + client.identify()?; + Ok(Self { stream: client.stream()? }) + } +} + +impl Stream for Chat { + type Item = irc::error::Result; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let next = self.stream.poll_next_unpin(cx); + match next { + Poll::Ready(Some(Ok(r))) => match self::event::to_chat_event(r) { + Some(ev) => Poll::Ready(Some(Ok(ev))), + None => { + cx.waker().wake_by_ref(); + Poll::Pending + } + }, + Poll::Ready(Some(Err(e))) => Poll::Ready(Some(Err(e))), + Poll::Ready(None) => Poll::Ready(None), + Poll::Pending => Poll::Pending + } + } +} diff --git a/src/youtube/mod.rs b/src/youtube/mod.rs new file mode 100644 index 0000000..2091d40 --- /dev/null +++ b/src/youtube/mod.rs @@ -0,0 +1,323 @@ +use std::{ + collections::{HashMap, VecDeque}, + io::BufRead, + sync::OnceLock, + time::{Instant, SystemTime, UNIX_EPOCH} +}; + +use regex::Regex; +use reqwest::{ + get, + header::{self, HeaderMap, HeaderValue, CONTENT_TYPE}, + StatusCode +}; +use simd_json::{ + base::{ValueAsContainer, ValueAsScalar}, + OwnedValue +}; +use thiserror::Error; +use tokio::sync::Mutex; + +mod types; +mod util; +use self::{ + types::{Action, GetLiveChatBody, GetLiveChatResponse, MessageRun}, + util::{SimdJsonRequestBody, SimdJsonResponseBody} +}; + +const GCM_SIGNALER_SRQE: &str = "https://signaler-pa.youtube.com/punctual/v1/chooseServer"; +const GCM_SIGNALER_PSUB: &str = "https://signaler-pa.youtube.com/punctual/multi-watch/channel"; + +const LIVE_CHAT_BASE_TANGO_KEY: &str = "AIzaSyDZNkyC-AtROwMBpLfevIvqYk-Gfi8ZOeo"; + +static USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:123.0) Gecko/20100101 Firefox/123.0"; +static HTTP_CLIENT: OnceLock = OnceLock::new(); + +#[derive(Debug, Error)] +pub enum YouTubeError { + #[error("impossible regex error")] + Regex(#[from] regex::Error), + #[error("error when deserializing: {0}")] + Deserialization(#[from] simd_json::Error), + #[error("missing continuation contents")] + MissingContinuationContents, + #[error("reached end of continuation")] + EndOfContinuation, + #[error("request timed out")] + TimedOut, + #[error("request returned bad HTTP status: {0}")] + BadStatus(StatusCode), + #[error("request error: {0}")] + GeneralRequest(reqwest::Error), + #[error("{0} is not a live stream")] + NotStream(String), + #[error("Failed to match InnerTube API key")] + NoInnerTubeKey, + #[error("Chat continuation token could not be found.")] + NoChatContinuation +} + +impl From for YouTubeError { + fn from(value: reqwest::Error) -> Self { + if value.is_timeout() { + YouTubeError::TimedOut + } else if value.is_status() { + YouTubeError::BadStatus(value.status().unwrap()) + } else { + YouTubeError::GeneralRequest(value) + } + } +} + +pub(crate) fn get_http_client() -> &'static reqwest::Client { + HTTP_CLIENT.get_or_init(|| { + let mut headers = HeaderMap::new(); + // Set our Accept-Language to en-US so we can properly match substrings + headers.append(header::ACCEPT_LANGUAGE, HeaderValue::from_static("en-US,en;q=0.5")); + headers.append(header::USER_AGENT, HeaderValue::from_static(USER_AGENT)); + reqwest::Client::builder().default_headers(headers).build().unwrap() + }) +} + +#[derive(Clone, Debug)] +pub struct RequestOptions { + pub api_key: String, + pub client_version: String, + pub live_status: bool +} + +pub async fn get_options_from_live_page(live_id: impl AsRef) -> Result<(RequestOptions, String), YouTubeError> { + let live_id = live_id.as_ref(); + let page_contents = get_http_client() + .get(format!("https://www.youtube.com/watch?v={live_id}")) + .send() + .await? + .text() + .await?; + + let live_status: bool; + let live_now_regex = Regex::new(r#"['"]isLiveNow['"]:\s*(true)"#)?; + let not_replay_regex = Regex::new(r#"['"]isReplay['"]:\s*(true)"#)?; + if live_now_regex.find(&page_contents).is_some() { + live_status = true; + } else if not_replay_regex.find(&page_contents).is_some() { + live_status = false; + } else { + return Err(YouTubeError::NotStream(live_id.to_string())); + } + + let api_key_regex = Regex::new(r#"['"]INNERTUBE_API_KEY['"]:\s*['"](.+?)['"]"#).unwrap(); + let api_key = match api_key_regex.captures(&page_contents).and_then(|captures| captures.get(1)) { + Some(matched) => matched.as_str().to_string(), + None => return Err(YouTubeError::NoInnerTubeKey) + }; + + let client_version_regex = Regex::new(r#"['"]clientVersion['"]:\s*['"]([\d.]+?)['"]"#).unwrap(); + let client_version = match client_version_regex.captures(&page_contents).and_then(|captures| captures.get(1)) { + Some(matched) => matched.as_str().to_string(), + None => "2.20230801.08.00".to_string() + }; + + let continuation_regex = if live_status { + Regex::new( + r#"Live chat['"],\s*['"]selected['"]:\s*(?:true|false),\s*['"]continuation['"]:\s*\{\s*['"]reloadContinuationData['"]:\s*\{['"]continuation['"]:\s*['"](.+?)['"]"# + )? + } else { + Regex::new( + r#"Top chat replay['"],\s*['"]selected['"]:\s*true,\s*['"]continuation['"]:\s*\{\s*['"]reloadContinuationData['"]:\s*\{['"]continuation['"]:\s*['"](.+?)['"]"# + )? + }; + let continuation = match continuation_regex.captures(&page_contents).and_then(|captures| captures.get(1)) { + Some(matched) => matched.as_str().to_string(), + None => return Err(YouTubeError::NoChatContinuation) + }; + + Ok((RequestOptions { api_key, client_version, live_status }, continuation)) +} +pub struct Author { + pub display_name: String, + pub id: String, + pub avatar: String +} + +pub struct ChatMessage { + pub runs: Vec, + pub is_super: bool, + pub author: Author, + pub timestamp: i64, + pub time_delta: i64 +} + +pub struct YouTubeChatPageProcessor<'r> { + actions: Mutex>, + request_options: &'r RequestOptions, + continuation_token: Option +} + +#[derive(Debug, Error)] +#[error("no continuation available")] +pub struct NoContinuationError; + +unsafe impl<'r> Send for YouTubeChatPageProcessor<'r> {} + +impl<'r> YouTubeChatPageProcessor<'r> { + pub fn new(response: GetLiveChatResponse, request_options: &'r RequestOptions, continuation_token: Option) -> Result { + Ok(Self { + actions: Mutex::new(VecDeque::from( + response + .continuation_contents + .ok_or(YouTubeError::MissingContinuationContents)? + .live_chat_continuation + .actions + .ok_or(YouTubeError::EndOfContinuation)? + )), + request_options, + continuation_token + }) + } +} + +impl<'r> Iterator for &YouTubeChatPageProcessor<'r> { + type Item = ChatMessage; + + fn next(&mut self) -> Option { + let mut next_action = None; + while next_action.is_none() { + match self.actions.try_lock().unwrap().pop_front() { + Some(action) => { + if let Some(replay) = action.replay_chat_item_action { + for action in replay.actions { + if next_action.is_some() { + break; + } + + if let Some(add_chat_item_action) = action.add_chat_item_action { + if let Some(text_message_renderer) = &add_chat_item_action.item.live_chat_text_message_renderer { + if text_message_renderer.message.is_some() { + next_action.replace((add_chat_item_action, replay.video_offset_time_msec)); + } + } else if let Some(superchat_renderer) = &add_chat_item_action.item.live_chat_paid_message_renderer { + if superchat_renderer.live_chat_text_message_renderer.message.is_some() { + next_action.replace((add_chat_item_action, replay.video_offset_time_msec)); + } + } + } + } + } + } + None => return None + } + } + + let (next_action, time_delta) = next_action.unwrap(); + let is_super = next_action.item.live_chat_paid_message_renderer.is_some(); + let renderer = if let Some(renderer) = next_action.item.live_chat_text_message_renderer { + renderer + } else if let Some(renderer) = next_action.item.live_chat_paid_message_renderer { + renderer.live_chat_text_message_renderer + } else { + panic!() + }; + + Some(ChatMessage { + runs: renderer.message.unwrap().runs, + is_super, + author: Author { + display_name: renderer + .message_renderer_base + .author_name + .map(|x| x.simple_text) + .unwrap_or_else(|| renderer.message_renderer_base.author_external_channel_id.to_owned()), + id: renderer.message_renderer_base.author_external_channel_id.to_owned(), + avatar: renderer.message_renderer_base.author_photo.thumbnails[renderer.message_renderer_base.author_photo.thumbnails.len() - 1] + .url + .to_owned() + }, + timestamp: renderer.message_renderer_base.timestamp_usec.timestamp_millis(), + time_delta + }) + } +} + +pub async fn fetch_yt_chat_page(options: &RequestOptions, continuation: impl AsRef) -> Result { + let url = + format!("https://www.youtube.com/youtubei/v1/live_chat/get_live_chat{}?key={}", if !options.live_status { "_replay" } else { "" }, &options.api_key); + let body = GetLiveChatBody::new(continuation.as_ref(), &options.client_version, "WEB"); + let response = get_http_client().post(url).simd_json(&body)?.send().await?; + let response: GetLiveChatResponse = response.simd_json().await?; + Ok(response) +} + +pub async fn subscribe_to_events(options: &RequestOptions, initial_continuation: &GetLiveChatResponse) -> Result<(), YouTubeError> { + let topic_id = &initial_continuation + .continuation_contents + .as_ref() + .unwrap() + .live_chat_continuation + .continuations[0] + .invalidation_continuation_data + .as_ref() + .unwrap() + .invalidation_id + .topic; + + let server_response: OwnedValue = get_http_client() + .post(format!("{GCM_SIGNALER_SRQE}?key={}", LIVE_CHAT_BASE_TANGO_KEY)) + .header(header::CONTENT_TYPE, "application/json+protobuf") + .header(header::REFERER, "https://www.youtube.com/") + .header("Sec-Fetch-Site", "same-site") + .header(header::ORIGIN, "https://www.youtube.com/") + .header(header::ACCEPT_ENCODING, "gzip, deflate, br") + .simd_json(&simd_json::json!([[null, null, null, [7, 5], null, [["youtube_live_chat_web"], [1], [[[&topic_id]]]]]]))? + .send() + .await? + .simd_json() + .await?; + let gsess = server_response.as_array().unwrap()[0].as_str().unwrap(); + + let mut ofs_parameters = HashMap::new(); + ofs_parameters.insert("count", "1".to_string()); + ofs_parameters.insert("ofs", "0".to_string()); + ofs_parameters.insert( + "req0___data__", + format!( + r#"[[["1",[null,null,null,[7,5],null,[["youtube_live_chat_web"],[1],[[["{}"]]]],null,null,1],null,3]]]"#, + &topic_id + ) + ); + let ofs = get_http_client() + .post(format!("{GCM_SIGNALER_PSUB}?VER=8&gsessionid={gsess}&key={LIVE_CHAT_BASE_TANGO_KEY}&RID=60464&CVER=22&zx=uo5vp9j380ef&t=1")) + .header(header::REFERER, "https://www.youtube.com/") + .header("Sec-Fetch-Site", "same-site") + .header(header::ORIGIN, "https://www.youtube.com/") + .header(header::ACCEPT_ENCODING, "gzip, deflate, br") + .header("X-WebChannel-Content-Type", "application/json+protobuf") + .form(&ofs_parameters) + .send() + .await?; + + let mut ofs_res_line = ofs.bytes().await?.lines().nth(1).unwrap().unwrap(); + let value: OwnedValue = unsafe { simd_json::from_str(&mut ofs_res_line) }?; + let value = value.as_array().unwrap()[0].as_array().unwrap(); + assert_eq!(value[0].as_usize().unwrap(), 0); + let sid = value[1].as_array().unwrap()[1].as_str().unwrap(); + + let mut stream = get_http_client() + .get(format!( + "{GCM_SIGNALER_PSUB}?VER=8&gsessionid={gsess}&key={LIVE_CHAT_BASE_TANGO_KEY}&RID=rpc&SID={sid}&AID=0&CI=0&TYPE=xmlhttp&zx=uo5vp9j380ed&t=1" + )) + .header(header::REFERER, "https://www.youtube.com/") + .header("Sec-Fetch-Site", "same-site") + .header(header::ORIGIN, "https://www.youtube.com/") + .header(header::ACCEPT_ENCODING, "gzip, deflate, br") + .header(header::ACCEPT, "*/*") + .header(header::CONNECTION, "keep-alive") + .send() + .await?; + + while let Some(c) = stream.chunk().await? { + println!("{}", String::from_utf8_lossy(&c)); + } + + Ok(()) +} diff --git a/src/youtube/types.rs b/src/youtube/types.rs new file mode 100644 index 0000000..182b1b8 --- /dev/null +++ b/src/youtube/types.rs @@ -0,0 +1,319 @@ +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use serde_aux::prelude::*; + +#[derive(Serialize, Debug)] +pub struct GetLiveChatBody { + context: GetLiveChatBodyContext, + continuation: String +} + +impl GetLiveChatBody { + pub fn new(continuation: impl Into, client_version: impl Into, client_name: impl Into) -> Self { + Self { + context: GetLiveChatBodyContext { + client: GetLiveChatBodyContextClient { + client_version: client_version.into(), + client_name: client_name.into() + } + }, + continuation: continuation.into() + } + } +} + +#[derive(Serialize, Debug)] +pub struct GetLiveChatBodyContext { + client: GetLiveChatBodyContextClient +} + +#[derive(Serialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct GetLiveChatBodyContextClient { + client_version: String, + client_name: String +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct GetLiveChatResponse { + pub response_context: Option, + pub tracking_params: Option, + pub continuation_contents: Option +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct GetLiveChatResponseContinuationContents { + pub live_chat_continuation: LiveChatContinuation +} +#[derive(Deserialize, Debug)] +pub struct LiveChatContinuation { + pub continuations: Vec, + pub actions: Option> +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct Continuation { + pub invalidation_continuation_data: Option, + pub timed_continuation_data: Option, + pub live_chat_replay_continuation_data: Option +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct LiveChatReplayContinuationData { + pub time_until_last_message_msec: usize, + pub continuation: String +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct InvalidationContinuationData { + pub invalidation_id: InvalidationId, + pub timeout_ms: usize, + pub continuation: String +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct InvalidationId { + pub object_source: usize, + pub object_id: String, + pub topic: String, + pub subscribe_to_gcm_topics: bool, + pub proto_creation_timestamp_ms: String +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct TimedContinuationData { + pub timeout_ms: usize, + pub continuation: String, + pub click_tracking_params: Option +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct Action { + pub add_chat_item_action: Option, + pub add_live_chat_ticker_item_action: Option, + pub replay_chat_item_action: Option +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct ReplayChatItemAction { + pub actions: Vec, + #[serde(deserialize_with = "deserialize_number_from_string")] + pub video_offset_time_msec: i64 +} + +// MessageRun +#[derive(Deserialize, Debug, Clone)] +#[serde(untagged)] +pub enum MessageRun { + MessageText { + text: String + }, + #[serde(rename_all = "camelCase")] + MessageEmoji { + emoji: Emoji, + variant_ids: Option> + } +} + +#[derive(Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct Emoji { + pub emoji_id: String, + pub shortcuts: Option>, + pub search_terms: Option>, + pub supports_skin_tone: Option, + pub image: Image, + pub is_custom_emoji: Option +} + +#[derive(Deserialize, Debug, Clone)] +pub struct Image { + pub thumbnails: Vec, + pub accessibility: Accessibility +} + +#[derive(Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct Accessibility { + pub accessibility_data: AccessibilityData +} + +#[derive(Deserialize, Debug, Clone)] +pub struct AccessibilityData { + pub label: String +} + +#[derive(Deserialize, Debug, Clone)] +pub struct Thumbnail { + pub url: String, + pub width: Option, + pub height: Option +} + +#[derive(Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct AuthorBadge { + pub live_chat_author_badge_renderer: LiveChatAuthorBadgeRenderer +} + +#[derive(Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct LiveChatAuthorBadgeRenderer { + pub custom_thumbnail: Option, + pub icon: Option, + pub tooltip: String, + pub accessibility: Accessibility +} + +#[derive(Deserialize, Debug, Clone)] +pub struct CustomThumbnail { + pub thumbnails: Vec +} + +#[derive(Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct Icon { + pub icon_type: String +} + +#[derive(Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct MessageRendererBase { + pub author_name: Option, + pub author_photo: AuthorPhoto, + pub author_badges: Option>, + pub context_menu_endpoint: ContextMenuEndpoint, + pub id: String, + #[serde(deserialize_with = "deserialize_datetime_utc_from_milliseconds")] + pub timestamp_usec: DateTime, + pub author_external_channel_id: String, + pub context_menu_accessibility: Accessibility +} + +#[derive(Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct ContextMenuEndpoint { + pub click_tracking_params: Option, + pub command_metadata: CommandMetadata, + pub live_chat_item_context_menu_endpoint: LiveChatItemContextMenuEndpoint +} + +#[derive(Deserialize, Debug, Clone)] +pub struct LiveChatItemContextMenuEndpoint { + pub params: String +} + +#[derive(Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct CommandMetadata { + pub web_command_metadata: WebCommandMetadata +} + +#[derive(Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct WebCommandMetadata { + pub ignore_navigation: bool +} + +#[derive(Deserialize, Debug, Clone)] +pub struct AuthorPhoto { + pub thumbnails: Vec +} + +#[derive(Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct AuthorName { + pub simple_text: String +} + +#[derive(Deserialize, Debug)] +pub struct LiveChatTextMessageRenderer { + #[serde(flatten)] + pub message_renderer_base: MessageRendererBase, + pub message: Option +} + +#[derive(Deserialize, Debug)] +pub struct Message { + pub runs: Vec +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct LiveChatPaidMessageRenderer { + #[serde(flatten)] + pub live_chat_text_message_renderer: LiveChatTextMessageRenderer, + pub purchase_amount_text: PurchaseAmountText, + pub header_background_color: isize, + pub header_text_color: isize, + pub body_background_color: isize, + pub body_text_color: isize, + pub author_name_text_color: isize +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct LiveChatPaidStickerRenderer { + #[serde(flatten)] + pub message_renderer_base: MessageRendererBase, + pub purchase_amount_text: PurchaseAmountText, + pub sticker: Sticker, + pub money_chip_background_color: isize, + pub money_chip_text_color: isize, + pub sticker_display_width: isize, + pub sticker_display_height: isize, + pub background_color: isize, + pub author_name_text_color: isize +} + +#[derive(Deserialize, Debug)] +pub struct Sticker { + pub thumbnails: Vec, + pub accessibility: Accessibility +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct PurchaseAmountText { + pub simple_text: String +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct LiveChatMembershipItemRenderer { + #[serde(flatten)] + pub message_renderer_base: MessageRendererBase, + pub header_sub_text: Option, + pub author_badges: Option> +} + +#[derive(Deserialize, Debug)] +pub struct HeaderSubText { + pub runs: Vec +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct AddChatItemAction { + pub item: ActionItem, + pub client_id: Option +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct ActionItem { + pub live_chat_text_message_renderer: Option, + pub live_chat_paid_message_renderer: Option, + pub live_chat_membership_item_renderer: Option, + pub live_chat_paid_sticker_renderer: Option, + pub live_chat_viewer_engagement_message_renderer: Option +} diff --git a/src/youtube/util.rs b/src/youtube/util.rs new file mode 100644 index 0000000..98f57e2 --- /dev/null +++ b/src/youtube/util.rs @@ -0,0 +1,29 @@ +use std::future::Future; + +use reqwest::{RequestBuilder, Response}; +use serde::{de::DeserializeOwned, Serialize}; + +use super::YouTubeError; + +pub trait SimdJsonResponseBody { + fn simd_json(self) -> impl Future>; +} + +impl SimdJsonResponseBody for Response { + async fn simd_json(self) -> Result { + let mut full = self.bytes().await?.to_vec(); + Ok(simd_json::from_slice(&mut full)?) + } +} + +pub trait SimdJsonRequestBody { + fn simd_json(self, json: &T) -> Result + where + Self: Sized; +} + +impl SimdJsonRequestBody for RequestBuilder { + fn simd_json(self, json: &T) -> Result { + Ok(self.body(simd_json::to_vec(json)?)) + } +} From ea9a4f638ca797470cca343117975134f37518fa Mon Sep 17 00:00:00 2001 From: "Carson M." Date: Sat, 27 Jan 2024 00:12:45 -0600 Subject: [PATCH 2/9] cleanup a little --- Cargo.toml | 3 +- examples/youtube.rs | 2 +- src/youtube/mod.rs | 98 ++++++++++++++++++++++++++------------------- 3 files changed, 60 insertions(+), 43 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index d446949..ced98a3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,7 @@ serde-aux = { version = "4.4", optional = true } uuid = { version = "1.5", optional = true } reqwest = { version = "0.11", optional = true } simd-json = { version = "0.13", optional = true } +url = { version = "2.5", optional = true } regex = { version = "1.10", optional = true } [dev-dependencies] @@ -27,7 +28,7 @@ tokio = { version = "1", features = [ "rt", "rt-multi-thread", "macros", "net" ] [features] default = [ "tls-native", "twitch", "youtube" ] twitch = [ "dep:irc", "dep:uuid" ] -youtube = [ "dep:simd-json", "dep:reqwest", "dep:serde", "dep:regex", "dep:serde-aux" ] +youtube = [ "dep:simd-json", "dep:reqwest", "dep:serde", "dep:url", "dep:regex", "dep:serde-aux" ] serde = [ "dep:serde", "chrono/serde", "uuid?/serde" ] tls-native = [ "irc?/tls-native" ] #tls-rust = [ "irc?/tls-rust" ] diff --git a/examples/youtube.rs b/examples/youtube.rs index d558f87..7cb94ef 100644 --- a/examples/youtube.rs +++ b/examples/youtube.rs @@ -5,7 +5,7 @@ use futures_util::StreamExt; #[tokio::main] async fn main() -> anyhow::Result<()> { - let (options, cont) = youtube::get_options_from_live_page("e-5D_Shoozk").await?; + let (options, cont) = youtube::get_options_from_live_page("5Z5Sys8-tLs").await?; let initial_chat = youtube::fetch_yt_chat_page(&options, cont).await?; youtube::subscribe_to_events(&options, &initial_chat).await?; Ok(()) diff --git a/src/youtube/mod.rs b/src/youtube/mod.rs index 2091d40..62451a7 100644 --- a/src/youtube/mod.rs +++ b/src/youtube/mod.rs @@ -1,14 +1,12 @@ use std::{ collections::{HashMap, VecDeque}, io::BufRead, - sync::OnceLock, - time::{Instant, SystemTime, UNIX_EPOCH} + sync::OnceLock }; use regex::Regex; use reqwest::{ - get, - header::{self, HeaderMap, HeaderValue, CONTENT_TYPE}, + header::{self, HeaderMap, HeaderValue}, StatusCode }; use simd_json::{ @@ -17,6 +15,7 @@ use simd_json::{ }; use thiserror::Error; use tokio::sync::Mutex; +use url::Url; mod types; mod util; @@ -27,11 +26,12 @@ use self::{ const GCM_SIGNALER_SRQE: &str = "https://signaler-pa.youtube.com/punctual/v1/chooseServer"; const GCM_SIGNALER_PSUB: &str = "https://signaler-pa.youtube.com/punctual/multi-watch/channel"; +const TANGO_LIVE_ENDPOINT: &str = "https://www.youtube.com/youtubei/v1/live_chat/get_live_chat"; +const TANGO_REPLAY_ENDPOINT: &str = "https://www.youtube.com/youtubei/v1/live_chat/get_live_chat_replay"; const LIVE_CHAT_BASE_TANGO_KEY: &str = "AIzaSyDZNkyC-AtROwMBpLfevIvqYk-Gfi8ZOeo"; -static USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:123.0) Gecko/20100101 Firefox/123.0"; -static HTTP_CLIENT: OnceLock = OnceLock::new(); +const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:123.0) Gecko/20100101 Firefox/123.0"; #[derive(Debug, Error)] pub enum YouTubeError { @@ -54,7 +54,9 @@ pub enum YouTubeError { #[error("Failed to match InnerTube API key")] NoInnerTubeKey, #[error("Chat continuation token could not be found.")] - NoChatContinuation + NoChatContinuation, + #[error("Error parsing URL: {0}")] + URLParseError(#[from] url::ParseError) } impl From for YouTubeError { @@ -70,11 +72,13 @@ impl From for YouTubeError { } pub(crate) fn get_http_client() -> &'static reqwest::Client { + static HTTP_CLIENT: OnceLock = OnceLock::new(); HTTP_CLIENT.get_or_init(|| { let mut headers = HeaderMap::new(); // Set our Accept-Language to en-US so we can properly match substrings headers.append(header::ACCEPT_LANGUAGE, HeaderValue::from_static("en-US,en;q=0.5")); headers.append(header::USER_AGENT, HeaderValue::from_static(USER_AGENT)); + headers.append(header::REFERER, HeaderValue::from_static("https://www.youtube.com/")); reqwest::Client::builder().default_headers(headers).build().unwrap() }) } @@ -154,10 +158,6 @@ pub struct YouTubeChatPageProcessor<'r> { continuation_token: Option } -#[derive(Debug, Error)] -#[error("no continuation available")] -pub struct NoContinuationError; - unsafe impl<'r> Send for YouTubeChatPageProcessor<'r> {} impl<'r> YouTubeChatPageProcessor<'r> { @@ -240,11 +240,17 @@ impl<'r> Iterator for &YouTubeChatPageProcessor<'r> { } pub async fn fetch_yt_chat_page(options: &RequestOptions, continuation: impl AsRef) -> Result { - let url = - format!("https://www.youtube.com/youtubei/v1/live_chat/get_live_chat{}?key={}", if !options.live_status { "_replay" } else { "" }, &options.api_key); let body = GetLiveChatBody::new(continuation.as_ref(), &options.client_version, "WEB"); - let response = get_http_client().post(url).simd_json(&body)?.send().await?; - let response: GetLiveChatResponse = response.simd_json().await?; + let response: GetLiveChatResponse = get_http_client() + .post(Url::parse_with_params( + if options.live_status { TANGO_LIVE_ENDPOINT } else { TANGO_REPLAY_ENDPOINT }, + [("key", options.api_key.as_str()), ("prettyPrint", "false")] + )?) + .simd_json(&body)? + .send() + .await? + .simd_json() + .await?; Ok(response) } @@ -262,13 +268,9 @@ pub async fn subscribe_to_events(options: &RequestOptions, initial_continuation: .topic; let server_response: OwnedValue = get_http_client() - .post(format!("{GCM_SIGNALER_SRQE}?key={}", LIVE_CHAT_BASE_TANGO_KEY)) + .post(Url::parse_with_params(GCM_SIGNALER_SRQE, [("key", LIVE_CHAT_BASE_TANGO_KEY)])?) .header(header::CONTENT_TYPE, "application/json+protobuf") - .header(header::REFERER, "https://www.youtube.com/") - .header("Sec-Fetch-Site", "same-site") - .header(header::ORIGIN, "https://www.youtube.com/") - .header(header::ACCEPT_ENCODING, "gzip, deflate, br") - .simd_json(&simd_json::json!([[null, null, null, [7, 5], null, [["youtube_live_chat_web"], [1], [[[&topic_id]]]]]]))? + .body(format!(r#"[[null,null,null,[7,5],null,[["youtube_live_chat_web"],[1],[[["{topic_id}"]]]]]]"#)) .send() .await? .simd_json() @@ -278,24 +280,28 @@ pub async fn subscribe_to_events(options: &RequestOptions, initial_continuation: let mut ofs_parameters = HashMap::new(); ofs_parameters.insert("count", "1".to_string()); ofs_parameters.insert("ofs", "0".to_string()); - ofs_parameters.insert( - "req0___data__", - format!( - r#"[[["1",[null,null,null,[7,5],null,[["youtube_live_chat_web"],[1],[[["{}"]]]],null,null,1],null,3]]]"#, - &topic_id - ) - ); + ofs_parameters + .insert("req0___data__", format!(r#"[[["1",[null,null,null,[7,5],null,[["youtube_live_chat_web"],[1],[[["{topic_id}"]]]],null,null,1],null,3]]]"#)); let ofs = get_http_client() - .post(format!("{GCM_SIGNALER_PSUB}?VER=8&gsessionid={gsess}&key={LIVE_CHAT_BASE_TANGO_KEY}&RID=60464&CVER=22&zx=uo5vp9j380ef&t=1")) - .header(header::REFERER, "https://www.youtube.com/") - .header("Sec-Fetch-Site", "same-site") - .header(header::ORIGIN, "https://www.youtube.com/") - .header(header::ACCEPT_ENCODING, "gzip, deflate, br") + .post(Url::parse_with_params( + GCM_SIGNALER_PSUB, + [ + ("VER", "8"), + ("gsessionid", &gsess), + ("key", LIVE_CHAT_BASE_TANGO_KEY), + ("RID", "60464"), + ("CVER", "22"), + ("zx", "uo5vp9j380ef"), + ("t", "1") + ] + )?) .header("X-WebChannel-Content-Type", "application/json+protobuf") .form(&ofs_parameters) .send() .await?; + // standard response: [[0,["c","koBtCISzwmqJpalH1EqHSc","",8,12,30000]]] + // stream end response: [1,3,7] let mut ofs_res_line = ofs.bytes().await?.lines().nth(1).unwrap().unwrap(); let value: OwnedValue = unsafe { simd_json::from_str(&mut ofs_res_line) }?; let value = value.as_array().unwrap()[0].as_array().unwrap(); @@ -303,14 +309,21 @@ pub async fn subscribe_to_events(options: &RequestOptions, initial_continuation: let sid = value[1].as_array().unwrap()[1].as_str().unwrap(); let mut stream = get_http_client() - .get(format!( - "{GCM_SIGNALER_PSUB}?VER=8&gsessionid={gsess}&key={LIVE_CHAT_BASE_TANGO_KEY}&RID=rpc&SID={sid}&AID=0&CI=0&TYPE=xmlhttp&zx=uo5vp9j380ed&t=1" - )) - .header(header::REFERER, "https://www.youtube.com/") - .header("Sec-Fetch-Site", "same-site") - .header(header::ORIGIN, "https://www.youtube.com/") - .header(header::ACCEPT_ENCODING, "gzip, deflate, br") - .header(header::ACCEPT, "*/*") + .get(Url::parse_with_params( + GCM_SIGNALER_PSUB, + [ + ("VER", "8"), + ("gsessionid", &gsess), + ("key", LIVE_CHAT_BASE_TANGO_KEY), + ("RID", "rpc"), + ("SID", sid), + ("AID", "0"), + ("CI", "0"), + ("TYPE", "xmlhttp"), + ("zx", "uo5vp9j380ef"), + ("t", "1") + ] + )?) .header(header::CONNECTION, "keep-alive") .send() .await?; @@ -319,5 +332,8 @@ pub async fn subscribe_to_events(options: &RequestOptions, initial_continuation: println!("{}", String::from_utf8_lossy(&c)); } + // todo: how to distinguish normal closure from server shutdown (NS_BINDING_ABORTED)? + println!("{:?}", stream.bytes().await); + Ok(()) } From b3baa2af90c61fbe9c4cd3a4d67de2a901057a3c Mon Sep 17 00:00:00 2001 From: "Carson M." Date: Sat, 27 Jan 2024 14:58:55 -0600 Subject: [PATCH 3/9] fix tests --- src/twitch/identity.rs | 2 +- src/twitch/mod.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/twitch/identity.rs b/src/twitch/identity.rs index d70c19f..74767e8 100644 --- a/src/twitch/identity.rs +++ b/src/twitch/identity.rs @@ -23,7 +23,7 @@ impl TwitchIdentity for Anonymous { /// use [`Anonymous`] instead. /// /// ```no_run -/// use brainrot::{Authenticated, Chat}; +/// use brainrot::twitch::{Authenticated, Chat}; /// /// # #[tokio::main] /// # async fn main() -> anyhow::Result<()> { diff --git a/src/twitch/mod.rs b/src/twitch/mod.rs index 2d90961..d93bfda 100644 --- a/src/twitch/mod.rs +++ b/src/twitch/mod.rs @@ -33,7 +33,7 @@ impl Chat { /// Connect to a Twitch IRC channel. /// /// ```no_run - /// use brainrot::{Anonymous, Chat}; + /// use brainrot::twitch::{Anonymous, Chat}; /// /// # #[tokio::main] /// # async fn main() -> anyhow::Result<()> { From 651d385371c280ed28a5a3e6995641bc7dc322b4 Mon Sep 17 00:00:00 2001 From: "Carson M." Date: Sat, 27 Jan 2024 21:18:47 -0600 Subject: [PATCH 4/9] more testing --- Cargo.toml | 3 +- examples/youtube.rs | 8 +- src/youtube/mod.rs | 293 ++++++++++++++++++++++++++++++++------------ 3 files changed, 220 insertions(+), 84 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index ced98a3..2f72869 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ uuid = { version = "1.5", optional = true } reqwest = { version = "0.11", optional = true } simd-json = { version = "0.13", optional = true } url = { version = "2.5", optional = true } +rand = { version = "0.8", optional = true } regex = { version = "1.10", optional = true } [dev-dependencies] @@ -28,7 +29,7 @@ tokio = { version = "1", features = [ "rt", "rt-multi-thread", "macros", "net" ] [features] default = [ "tls-native", "twitch", "youtube" ] twitch = [ "dep:irc", "dep:uuid" ] -youtube = [ "dep:simd-json", "dep:reqwest", "dep:serde", "dep:url", "dep:regex", "dep:serde-aux" ] +youtube = [ "dep:simd-json", "dep:reqwest", "dep:rand", "dep:serde", "dep:url", "dep:regex", "dep:serde-aux" ] serde = [ "dep:serde", "chrono/serde", "uuid?/serde" ] tls-native = [ "irc?/tls-native" ] #tls-rust = [ "irc?/tls-rust" ] diff --git a/examples/youtube.rs b/examples/youtube.rs index 7cb94ef..29b4827 100644 --- a/examples/youtube.rs +++ b/examples/youtube.rs @@ -1,12 +1,14 @@ -use std::env::args; +use std::{env::args, future::IntoFuture}; use brainrot::youtube; use futures_util::StreamExt; #[tokio::main] async fn main() -> anyhow::Result<()> { - let (options, cont) = youtube::get_options_from_live_page("5Z5Sys8-tLs").await?; + let (options, cont) = youtube::get_options_from_live_page("6DcXroWNDvk").await?; let initial_chat = youtube::fetch_yt_chat_page(&options, cont).await?; - youtube::subscribe_to_events(&options, &initial_chat).await?; + let subscriber = youtube::SignalerChannel::new_from_cont(&initial_chat).await?; + let (receiver, handle) = subscriber.spawn_event_subscriber().await?; + handle.into_future().await.unwrap(); Ok(()) } diff --git a/src/youtube/mod.rs b/src/youtube/mod.rs index 62451a7..9557fec 100644 --- a/src/youtube/mod.rs +++ b/src/youtube/mod.rs @@ -1,20 +1,25 @@ use std::{ collections::{HashMap, VecDeque}, io::BufRead, - sync::OnceLock + iter, + sync::{Arc, OnceLock} }; +use rand::Rng; use regex::Regex; use reqwest::{ header::{self, HeaderMap, HeaderValue}, - StatusCode + Response, StatusCode }; use simd_json::{ base::{ValueAsContainer, ValueAsScalar}, OwnedValue }; use thiserror::Error; -use tokio::sync::Mutex; +use tokio::{ + sync::{broadcast, Mutex}, + task::JoinHandle +}; use url::Url; mod types; @@ -254,86 +259,214 @@ pub async fn fetch_yt_chat_page(options: &RequestOptions, continuation: impl AsR Ok(response) } -pub async fn subscribe_to_events(options: &RequestOptions, initial_continuation: &GetLiveChatResponse) -> Result<(), YouTubeError> { - let topic_id = &initial_continuation - .continuation_contents - .as_ref() - .unwrap() - .live_chat_continuation - .continuations[0] - .invalidation_continuation_data - .as_ref() - .unwrap() - .invalidation_id - .topic; - - let server_response: OwnedValue = get_http_client() - .post(Url::parse_with_params(GCM_SIGNALER_SRQE, [("key", LIVE_CHAT_BASE_TANGO_KEY)])?) - .header(header::CONTENT_TYPE, "application/json+protobuf") - .body(format!(r#"[[null,null,null,[7,5],null,[["youtube_live_chat_web"],[1],[[["{topic_id}"]]]]]]"#)) - .send() - .await? - .simd_json() - .await?; - let gsess = server_response.as_array().unwrap()[0].as_str().unwrap(); - - let mut ofs_parameters = HashMap::new(); - ofs_parameters.insert("count", "1".to_string()); - ofs_parameters.insert("ofs", "0".to_string()); - ofs_parameters - .insert("req0___data__", format!(r#"[[["1",[null,null,null,[7,5],null,[["youtube_live_chat_web"],[1],[[["{topic_id}"]]]],null,null,1],null,3]]]"#)); - let ofs = get_http_client() - .post(Url::parse_with_params( - GCM_SIGNALER_PSUB, - [ - ("VER", "8"), - ("gsessionid", &gsess), - ("key", LIVE_CHAT_BASE_TANGO_KEY), - ("RID", "60464"), - ("CVER", "22"), - ("zx", "uo5vp9j380ef"), - ("t", "1") - ] - )?) - .header("X-WebChannel-Content-Type", "application/json+protobuf") - .form(&ofs_parameters) - .send() - .await?; +#[derive(Debug, Default)] +struct SignalerChannelInner { + topic: String, + gsessionid: Option, + sid: Option, + rid: usize, + aid: usize, + session_n: usize +} - // standard response: [[0,["c","koBtCISzwmqJpalH1EqHSc","",8,12,30000]]] - // stream end response: [1,3,7] - let mut ofs_res_line = ofs.bytes().await?.lines().nth(1).unwrap().unwrap(); - let value: OwnedValue = unsafe { simd_json::from_str(&mut ofs_res_line) }?; - let value = value.as_array().unwrap()[0].as_array().unwrap(); - assert_eq!(value[0].as_usize().unwrap(), 0); - let sid = value[1].as_array().unwrap()[1].as_str().unwrap(); - - let mut stream = get_http_client() - .get(Url::parse_with_params( - GCM_SIGNALER_PSUB, - [ - ("VER", "8"), - ("gsessionid", &gsess), - ("key", LIVE_CHAT_BASE_TANGO_KEY), - ("RID", "rpc"), - ("SID", sid), - ("AID", "0"), - ("CI", "0"), - ("TYPE", "xmlhttp"), - ("zx", "uo5vp9j380ef"), - ("t", "1") - ] - )?) - .header(header::CONNECTION, "keep-alive") - .send() - .await?; +impl SignalerChannelInner { + pub fn with_topic(topic: impl ToString) -> Self { + Self { + topic: topic.to_string(), + ..Default::default() + } + } + + pub fn reset(&mut self) { + self.gsessionid = None; + self.sid = None; + self.rid = 0; + self.aid = 0; + self.session_n = 0; + } + + fn gen_zx() -> String { + const CHARSET: &[u8] = b"abcdefghijklmnopqrstuvwxyz0123456789"; + let mut rng = rand::thread_rng(); + iter::repeat_with(|| CHARSET[rng.gen_range(0..CHARSET.len())] as char).take(11).collect() + } + + pub async fn choose_server(&mut self) -> Result<(), YouTubeError> { + let server_response: OwnedValue = get_http_client() + .post(Url::parse_with_params(GCM_SIGNALER_SRQE, [("key", LIVE_CHAT_BASE_TANGO_KEY)])?) + .header(header::CONTENT_TYPE, "application/json+protobuf") + .body(format!(r#"[[null,null,null,[7,5],null,[["youtube_live_chat_web"],[1],[[["{}"]]]]]]"#, self.topic)) + .send() + .await? + .simd_json() + .await?; + let gsess = server_response.as_array().unwrap()[0].as_str().unwrap(); + self.gsessionid = Some(gsess.to_owned()); + Ok(()) + } - while let Some(c) = stream.chunk().await? { - println!("{}", String::from_utf8_lossy(&c)); + pub async fn renew_session_or_something(&mut self) -> Result<(), YouTubeError> { + let mut ofs_parameters = HashMap::new(); + ofs_parameters.insert("count", "2".to_string()); + ofs_parameters.insert("ofs", "1".to_string()); + ofs_parameters.insert("req0___data__", format!(r#"[[["{}",null,[]]]]"#, self.session_n)); + self.session_n += 1; + ofs_parameters.insert( + "req1___data__", + format!(r#"[[["{}",[null,null,null,[7,5],null,[["youtube_live_chat_web"],[1],[[["{}"]]]],null,null,1],null,3]]]"#, self.session_n, self.topic) + ); + let ofs = get_http_client() + .post(Url::parse_with_params( + GCM_SIGNALER_PSUB, + [ + ("VER", "8"), + ("gsessionid", self.gsessionid.as_ref().unwrap()), + ("key", LIVE_CHAT_BASE_TANGO_KEY), + ("SID", self.sid.as_ref().unwrap()), + ("RID", &self.rid.to_string()), + ("AID", &self.aid.to_string()), + ("CVER", "22"), + ("zx", Self::gen_zx().as_ref()), + ("t", "1") + ] + )?) + .header("X-WebChannel-Content-Type", "application/json+protobuf") + .form(&ofs_parameters) + .send() + .await?; + + let mut ofs_res_line = ofs.bytes().await?.lines().nth(1).unwrap().unwrap(); + println!("{ofs_res_line}"); + let value: OwnedValue = unsafe { simd_json::from_str(&mut ofs_res_line) }?; + let value = value.as_array().unwrap(); + // assert_eq!(value[0].as_usize().unwrap(), 1); + + Ok(()) } - // todo: how to distinguish normal closure from server shutdown (NS_BINDING_ABORTED)? - println!("{:?}", stream.bytes().await); + pub async fn init_session(&mut self) -> Result<(), YouTubeError> { + let mut ofs_parameters = HashMap::new(); + ofs_parameters.insert("count", "1".to_string()); + ofs_parameters.insert("ofs", "0".to_string()); + ofs_parameters.insert( + "req0___data__", + format!(r#"[[["1",[null,null,null,[7,5],null,[["youtube_live_chat_web"],[1],[[["{}"]]]],null,null,1],null,3]]]"#, self.topic) + ); + self.session_n = 1; + let ofs = get_http_client() + .post(Url::parse_with_params( + GCM_SIGNALER_PSUB, + [ + ("VER", "8"), + ("gsessionid", self.gsessionid.as_ref().unwrap()), + ("key", LIVE_CHAT_BASE_TANGO_KEY), + ("RID", &self.rid.to_string()), + ("AID", &self.aid.to_string()), + ("CVER", "22"), + ("zx", Self::gen_zx().as_ref()), + ("t", "1") + ] + )?) + .header("X-WebChannel-Content-Type", "application/json+protobuf") + .form(&ofs_parameters) + .send() + .await?; + + let mut ofs_res_line = ofs.bytes().await?.lines().nth(1).unwrap().unwrap(); + let value: OwnedValue = unsafe { simd_json::from_str(&mut ofs_res_line) }?; + let value = value.as_array().unwrap()[0].as_array().unwrap(); + assert_eq!(value[0].as_usize().unwrap(), 0); + let sid = value[1].as_array().unwrap()[1].as_str().unwrap(); + self.sid = Some(sid.to_owned()); + Ok(()) + } + + pub async fn get_session_stream(&self) -> Result { + Ok(get_http_client() + .get(Url::parse_with_params( + GCM_SIGNALER_PSUB, + [ + ("VER", "8"), + ("gsessionid", self.gsessionid.as_ref().unwrap()), + ("key", LIVE_CHAT_BASE_TANGO_KEY), + ("RID", "rpc"), + ("SID", self.sid.as_ref().unwrap()), + ("AID", &self.aid.to_string()), + ("CI", "0"), + ("TYPE", "xmlhttp"), + ("zx", &Self::gen_zx()), + ("t", "1") + ] + )?) + .header(header::CONNECTION, "keep-alive") + .send() + .await?) + } +} + +#[derive(Debug)] +pub struct SignalerChannel { + inner: Arc> +} - Ok(()) +impl SignalerChannel { + pub async fn new(topic_id: impl ToString) -> Result { + Ok(SignalerChannel { + inner: Arc::new(Mutex::new(SignalerChannelInner::with_topic(topic_id))) + }) + } + + pub async fn new_from_cont(cont: &GetLiveChatResponse) -> Result { + Ok(SignalerChannel { + inner: Arc::new(Mutex::new(SignalerChannelInner::with_topic( + &cont.continuation_contents.as_ref().unwrap().live_chat_continuation.continuations[0] + .invalidation_continuation_data + .as_ref() + .unwrap() + .invalidation_id + .topic + ))) + }) + } + + pub async fn spawn_event_subscriber(&self) -> Result<(broadcast::Receiver<()>, JoinHandle<()>), YouTubeError> { + let inner = Arc::clone(&self.inner); + { + let mut lock = inner.lock().await; + lock.choose_server().await?; + lock.init_session().await?; + } + let (sender, receiver) = broadcast::channel(128); + let handle = tokio::spawn(async move { + loop { + let mut req = { + let mut lock = inner.lock().await; + lock.reset(); + lock.choose_server().await.unwrap(); + lock.init_session().await.unwrap(); + lock.get_session_stream().await.unwrap() + }; + loop { + match req.chunk().await { + Ok(None) => break, + Ok(Some(s)) => { + let mut ofs_res_line = s.lines().nth(1).unwrap().unwrap(); + println!("{ofs_res_line}"); + if let Ok(s) = unsafe { simd_json::from_str::(ofs_res_line.as_mut()) } { + let a = s.as_array().unwrap(); + { + inner.lock().await.aid = a[a.len() - 1].as_array().unwrap()[0].as_usize().unwrap(); + } + } + } + Err(e) => { + eprintln!("{e:?}"); + break; + } + } + } + } + }); + Ok((receiver, handle)) + } } From 0b7fac66c1955756b3927b361953646aefcee7a8 Mon Sep 17 00:00:00 2001 From: "Carson M." Date: Sun, 28 Jan 2024 17:25:52 -0600 Subject: [PATCH 5/9] finally, something that works --- examples/youtube.rs | 50 +++++- src/youtube/mod.rs | 327 ++++++++++------------------------------ src/youtube/signaler.rs | 201 ++++++++++++++++++++++++ src/youtube/types.rs | 15 ++ 4 files changed, 341 insertions(+), 252 deletions(-) create mode 100644 src/youtube/signaler.rs diff --git a/examples/youtube.rs b/examples/youtube.rs index 29b4827..805abba 100644 --- a/examples/youtube.rs +++ b/examples/youtube.rs @@ -1,14 +1,48 @@ -use std::{env::args, future::IntoFuture}; +use std::future::IntoFuture; -use brainrot::youtube; -use futures_util::StreamExt; +use brainrot::youtube::{self, YouTubeChatPageProcessor}; #[tokio::main] async fn main() -> anyhow::Result<()> { - let (options, cont) = youtube::get_options_from_live_page("6DcXroWNDvk").await?; - let initial_chat = youtube::fetch_yt_chat_page(&options, cont).await?; - let subscriber = youtube::SignalerChannel::new_from_cont(&initial_chat).await?; - let (receiver, handle) = subscriber.spawn_event_subscriber().await?; - handle.into_future().await.unwrap(); + let (options, cont) = youtube::get_options_from_live_page("S144F6Cifyc").await?; + let initial_chat = youtube::fetch_yt_chat_page(&options, &cont).await?; + let topic = initial_chat.continuation_contents.as_ref().unwrap().live_chat_continuation.continuations[0] + .invalidation_continuation_data + .as_ref() + .unwrap() + .invalidation_id + .topic + .to_owned(); + let subscriber = youtube::SignalerChannel::new(topic).await?; + let (mut receiver, _handle) = subscriber.spawn_event_subscriber().await?; + tokio::spawn(async move { + let mut processor = YouTubeChatPageProcessor::new(initial_chat, &options).unwrap(); + for msg in &processor { + println!("{}: {}", msg.author.display_name, msg.runs.iter().map(|c| c.to_string()).collect::()); + } + + while receiver.recv().await.is_ok() { + match processor.cont().await { + Some(Ok(s)) => { + processor = s; + for msg in &processor { + println!("{}: {}", msg.author.display_name, msg.runs.iter().map(|c| c.to_string()).collect::()); + } + + subscriber.refresh_topic(processor.signaler_topic.as_ref().unwrap()).await; + } + Some(Err(e)) => { + eprintln!("{e:?}"); + break; + } + None => { + eprintln!("none"); + break; + } + } + } + }); + _handle.into_future().await.unwrap(); + println!("???"); Ok(()) } diff --git a/src/youtube/mod.rs b/src/youtube/mod.rs index 9557fec..9aca8d9 100644 --- a/src/youtube/mod.rs +++ b/src/youtube/mod.rs @@ -1,41 +1,26 @@ -use std::{ - collections::{HashMap, VecDeque}, - io::BufRead, - iter, - sync::{Arc, OnceLock} -}; +use std::{collections::VecDeque, sync::OnceLock}; -use rand::Rng; use regex::Regex; use reqwest::{ header::{self, HeaderMap, HeaderValue}, - Response, StatusCode -}; -use simd_json::{ - base::{ValueAsContainer, ValueAsScalar}, - OwnedValue + StatusCode }; use thiserror::Error; -use tokio::{ - sync::{broadcast, Mutex}, - task::JoinHandle -}; +use tokio::sync::Mutex; use url::Url; +mod signaler; mod types; mod util; +pub use self::signaler::SignalerChannel; use self::{ types::{Action, GetLiveChatBody, GetLiveChatResponse, MessageRun}, util::{SimdJsonRequestBody, SimdJsonResponseBody} }; -const GCM_SIGNALER_SRQE: &str = "https://signaler-pa.youtube.com/punctual/v1/chooseServer"; -const GCM_SIGNALER_PSUB: &str = "https://signaler-pa.youtube.com/punctual/multi-watch/channel"; const TANGO_LIVE_ENDPOINT: &str = "https://www.youtube.com/youtubei/v1/live_chat/get_live_chat"; const TANGO_REPLAY_ENDPOINT: &str = "https://www.youtube.com/youtubei/v1/live_chat/get_live_chat_replay"; -const LIVE_CHAT_BASE_TANGO_KEY: &str = "AIzaSyDZNkyC-AtROwMBpLfevIvqYk-Gfi8ZOeo"; - const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:123.0) Gecko/20100101 Firefox/123.0"; #[derive(Debug, Error)] @@ -90,9 +75,9 @@ pub(crate) fn get_http_client() -> &'static reqwest::Client { #[derive(Clone, Debug)] pub struct RequestOptions { - pub api_key: String, - pub client_version: String, - pub live_status: bool + pub(crate) api_key: String, + pub(crate) client_version: String, + pub(crate) live_status: bool } pub async fn get_options_from_live_page(live_id: impl AsRef) -> Result<(RequestOptions, String), YouTubeError> { @@ -154,32 +139,88 @@ pub struct ChatMessage { pub is_super: bool, pub author: Author, pub timestamp: i64, - pub time_delta: i64 + pub time_delta: Option } pub struct YouTubeChatPageProcessor<'r> { actions: Mutex>, request_options: &'r RequestOptions, - continuation_token: Option + continuation_token: Option, + pub signaler_topic: Option } unsafe impl<'r> Send for YouTubeChatPageProcessor<'r> {} impl<'r> YouTubeChatPageProcessor<'r> { - pub fn new(response: GetLiveChatResponse, request_options: &'r RequestOptions, continuation_token: Option) -> Result { + pub fn new(response: GetLiveChatResponse, request_options: &'r RequestOptions) -> Result { + let continuation_token = if request_options.live_status { + response + .continuation_contents + .as_ref() + .ok_or(YouTubeError::MissingContinuationContents)? + .live_chat_continuation + .continuations[0] + .invalidation_continuation_data + .as_ref() + .map(|x| x.continuation.to_owned()) + } else { + response + .continuation_contents + .as_ref() + .ok_or(YouTubeError::MissingContinuationContents)? + .live_chat_continuation + .continuations[0] + .live_chat_replay_continuation_data + .as_ref() + .map(|x| x.continuation.to_owned()) + }; + let signaler_topic = if request_options.live_status { + Some( + response.continuation_contents.as_ref().unwrap().live_chat_continuation.continuations[0] + .invalidation_continuation_data + .as_ref() + .unwrap() + .invalidation_id + .topic + .to_owned() + ) + } else { + None + }; Ok(Self { - actions: Mutex::new(VecDeque::from( + actions: Mutex::new(VecDeque::from(if request_options.live_status { + response + .continuation_contents + .ok_or(YouTubeError::MissingContinuationContents)? + .live_chat_continuation + .actions + .unwrap_or_default() + } else { response .continuation_contents .ok_or(YouTubeError::MissingContinuationContents)? .live_chat_continuation .actions .ok_or(YouTubeError::EndOfContinuation)? - )), + })), request_options, - continuation_token + continuation_token, + signaler_topic }) } + + async fn next_page(&self, continuation_token: &String) -> Result { + let page = fetch_yt_chat_page(self.request_options, continuation_token).await?; + YouTubeChatPageProcessor::new(page, self.request_options) + } + + pub async fn cont(&self) -> Option> { + if let Some(continuation_token) = &self.continuation_token { + Some(self.next_page(continuation_token).await) + } else { + None + } + } } impl<'r> Iterator for &YouTubeChatPageProcessor<'r> { @@ -199,15 +240,25 @@ impl<'r> Iterator for &YouTubeChatPageProcessor<'r> { if let Some(add_chat_item_action) = action.add_chat_item_action { if let Some(text_message_renderer) = &add_chat_item_action.item.live_chat_text_message_renderer { if text_message_renderer.message.is_some() { - next_action.replace((add_chat_item_action, replay.video_offset_time_msec)); + next_action.replace((add_chat_item_action, Some(replay.video_offset_time_msec))); } } else if let Some(superchat_renderer) = &add_chat_item_action.item.live_chat_paid_message_renderer { if superchat_renderer.live_chat_text_message_renderer.message.is_some() { - next_action.replace((add_chat_item_action, replay.video_offset_time_msec)); + next_action.replace((add_chat_item_action, Some(replay.video_offset_time_msec))); } } } } + } else if let Some(action) = action.add_chat_item_action { + if let Some(text_message_renderer) = &action.item.live_chat_text_message_renderer { + if text_message_renderer.message.is_some() { + next_action.replace((action, None)); + } + } else if let Some(superchat_renderer) = &action.item.live_chat_paid_message_renderer { + if superchat_renderer.live_chat_text_message_renderer.message.is_some() { + next_action.replace((action, None)); + } + } } } None => return None @@ -221,7 +272,7 @@ impl<'r> Iterator for &YouTubeChatPageProcessor<'r> { } else if let Some(renderer) = next_action.item.live_chat_paid_message_renderer { renderer.live_chat_text_message_renderer } else { - panic!() + unimplemented!() }; Some(ChatMessage { @@ -258,215 +309,3 @@ pub async fn fetch_yt_chat_page(options: &RequestOptions, continuation: impl AsR .await?; Ok(response) } - -#[derive(Debug, Default)] -struct SignalerChannelInner { - topic: String, - gsessionid: Option, - sid: Option, - rid: usize, - aid: usize, - session_n: usize -} - -impl SignalerChannelInner { - pub fn with_topic(topic: impl ToString) -> Self { - Self { - topic: topic.to_string(), - ..Default::default() - } - } - - pub fn reset(&mut self) { - self.gsessionid = None; - self.sid = None; - self.rid = 0; - self.aid = 0; - self.session_n = 0; - } - - fn gen_zx() -> String { - const CHARSET: &[u8] = b"abcdefghijklmnopqrstuvwxyz0123456789"; - let mut rng = rand::thread_rng(); - iter::repeat_with(|| CHARSET[rng.gen_range(0..CHARSET.len())] as char).take(11).collect() - } - - pub async fn choose_server(&mut self) -> Result<(), YouTubeError> { - let server_response: OwnedValue = get_http_client() - .post(Url::parse_with_params(GCM_SIGNALER_SRQE, [("key", LIVE_CHAT_BASE_TANGO_KEY)])?) - .header(header::CONTENT_TYPE, "application/json+protobuf") - .body(format!(r#"[[null,null,null,[7,5],null,[["youtube_live_chat_web"],[1],[[["{}"]]]]]]"#, self.topic)) - .send() - .await? - .simd_json() - .await?; - let gsess = server_response.as_array().unwrap()[0].as_str().unwrap(); - self.gsessionid = Some(gsess.to_owned()); - Ok(()) - } - - pub async fn renew_session_or_something(&mut self) -> Result<(), YouTubeError> { - let mut ofs_parameters = HashMap::new(); - ofs_parameters.insert("count", "2".to_string()); - ofs_parameters.insert("ofs", "1".to_string()); - ofs_parameters.insert("req0___data__", format!(r#"[[["{}",null,[]]]]"#, self.session_n)); - self.session_n += 1; - ofs_parameters.insert( - "req1___data__", - format!(r#"[[["{}",[null,null,null,[7,5],null,[["youtube_live_chat_web"],[1],[[["{}"]]]],null,null,1],null,3]]]"#, self.session_n, self.topic) - ); - let ofs = get_http_client() - .post(Url::parse_with_params( - GCM_SIGNALER_PSUB, - [ - ("VER", "8"), - ("gsessionid", self.gsessionid.as_ref().unwrap()), - ("key", LIVE_CHAT_BASE_TANGO_KEY), - ("SID", self.sid.as_ref().unwrap()), - ("RID", &self.rid.to_string()), - ("AID", &self.aid.to_string()), - ("CVER", "22"), - ("zx", Self::gen_zx().as_ref()), - ("t", "1") - ] - )?) - .header("X-WebChannel-Content-Type", "application/json+protobuf") - .form(&ofs_parameters) - .send() - .await?; - - let mut ofs_res_line = ofs.bytes().await?.lines().nth(1).unwrap().unwrap(); - println!("{ofs_res_line}"); - let value: OwnedValue = unsafe { simd_json::from_str(&mut ofs_res_line) }?; - let value = value.as_array().unwrap(); - // assert_eq!(value[0].as_usize().unwrap(), 1); - - Ok(()) - } - - pub async fn init_session(&mut self) -> Result<(), YouTubeError> { - let mut ofs_parameters = HashMap::new(); - ofs_parameters.insert("count", "1".to_string()); - ofs_parameters.insert("ofs", "0".to_string()); - ofs_parameters.insert( - "req0___data__", - format!(r#"[[["1",[null,null,null,[7,5],null,[["youtube_live_chat_web"],[1],[[["{}"]]]],null,null,1],null,3]]]"#, self.topic) - ); - self.session_n = 1; - let ofs = get_http_client() - .post(Url::parse_with_params( - GCM_SIGNALER_PSUB, - [ - ("VER", "8"), - ("gsessionid", self.gsessionid.as_ref().unwrap()), - ("key", LIVE_CHAT_BASE_TANGO_KEY), - ("RID", &self.rid.to_string()), - ("AID", &self.aid.to_string()), - ("CVER", "22"), - ("zx", Self::gen_zx().as_ref()), - ("t", "1") - ] - )?) - .header("X-WebChannel-Content-Type", "application/json+protobuf") - .form(&ofs_parameters) - .send() - .await?; - - let mut ofs_res_line = ofs.bytes().await?.lines().nth(1).unwrap().unwrap(); - let value: OwnedValue = unsafe { simd_json::from_str(&mut ofs_res_line) }?; - let value = value.as_array().unwrap()[0].as_array().unwrap(); - assert_eq!(value[0].as_usize().unwrap(), 0); - let sid = value[1].as_array().unwrap()[1].as_str().unwrap(); - self.sid = Some(sid.to_owned()); - Ok(()) - } - - pub async fn get_session_stream(&self) -> Result { - Ok(get_http_client() - .get(Url::parse_with_params( - GCM_SIGNALER_PSUB, - [ - ("VER", "8"), - ("gsessionid", self.gsessionid.as_ref().unwrap()), - ("key", LIVE_CHAT_BASE_TANGO_KEY), - ("RID", "rpc"), - ("SID", self.sid.as_ref().unwrap()), - ("AID", &self.aid.to_string()), - ("CI", "0"), - ("TYPE", "xmlhttp"), - ("zx", &Self::gen_zx()), - ("t", "1") - ] - )?) - .header(header::CONNECTION, "keep-alive") - .send() - .await?) - } -} - -#[derive(Debug)] -pub struct SignalerChannel { - inner: Arc> -} - -impl SignalerChannel { - pub async fn new(topic_id: impl ToString) -> Result { - Ok(SignalerChannel { - inner: Arc::new(Mutex::new(SignalerChannelInner::with_topic(topic_id))) - }) - } - - pub async fn new_from_cont(cont: &GetLiveChatResponse) -> Result { - Ok(SignalerChannel { - inner: Arc::new(Mutex::new(SignalerChannelInner::with_topic( - &cont.continuation_contents.as_ref().unwrap().live_chat_continuation.continuations[0] - .invalidation_continuation_data - .as_ref() - .unwrap() - .invalidation_id - .topic - ))) - }) - } - - pub async fn spawn_event_subscriber(&self) -> Result<(broadcast::Receiver<()>, JoinHandle<()>), YouTubeError> { - let inner = Arc::clone(&self.inner); - { - let mut lock = inner.lock().await; - lock.choose_server().await?; - lock.init_session().await?; - } - let (sender, receiver) = broadcast::channel(128); - let handle = tokio::spawn(async move { - loop { - let mut req = { - let mut lock = inner.lock().await; - lock.reset(); - lock.choose_server().await.unwrap(); - lock.init_session().await.unwrap(); - lock.get_session_stream().await.unwrap() - }; - loop { - match req.chunk().await { - Ok(None) => break, - Ok(Some(s)) => { - let mut ofs_res_line = s.lines().nth(1).unwrap().unwrap(); - println!("{ofs_res_line}"); - if let Ok(s) = unsafe { simd_json::from_str::(ofs_res_line.as_mut()) } { - let a = s.as_array().unwrap(); - { - inner.lock().await.aid = a[a.len() - 1].as_array().unwrap()[0].as_usize().unwrap(); - } - } - } - Err(e) => { - eprintln!("{e:?}"); - break; - } - } - } - } - }); - Ok((receiver, handle)) - } -} diff --git a/src/youtube/signaler.rs b/src/youtube/signaler.rs new file mode 100644 index 0000000..e89f45e --- /dev/null +++ b/src/youtube/signaler.rs @@ -0,0 +1,201 @@ +use std::{collections::HashMap, io::BufRead, iter, sync::Arc}; + +use rand::Rng; +use reqwest::{header, Response}; +use simd_json::{ + base::{ValueAsContainer, ValueAsScalar}, + OwnedValue +}; +use tokio::{ + sync::{broadcast, Mutex}, + task::JoinHandle +}; +use url::Url; + +use super::{types::GetLiveChatResponse, util::SimdJsonResponseBody, YouTubeError}; + +const GCM_SIGNALER_SRQE: &str = "https://signaler-pa.youtube.com/punctual/v1/chooseServer"; +const GCM_SIGNALER_PSUB: &str = "https://signaler-pa.youtube.com/punctual/multi-watch/channel"; + +const LIVE_CHAT_BASE_TANGO_KEY: &str = "AIzaSyDZNkyC-AtROwMBpLfevIvqYk-Gfi8ZOeo"; + +#[derive(Debug, Default)] +struct SignalerChannelInner { + topic: String, + gsessionid: Option, + sid: Option, + rid: usize, + aid: usize, + session_n: usize +} + +impl SignalerChannelInner { + pub fn with_topic(topic: impl ToString) -> Self { + Self { + topic: topic.to_string(), + ..Default::default() + } + } + + pub fn reset(&mut self) { + self.gsessionid = None; + self.sid = None; + self.rid = 0; + self.aid = 0; + self.session_n = 0; + } + + fn gen_zx() -> String { + const CHARSET: &[u8] = b"abcdefghijklmnopqrstuvwxyz0123456789"; + let mut rng = rand::thread_rng(); + iter::repeat_with(|| CHARSET[rng.gen_range(0..CHARSET.len())] as char).take(11).collect() + } + + pub async fn choose_server(&mut self) -> Result<(), YouTubeError> { + let server_response: OwnedValue = super::get_http_client() + .post(Url::parse_with_params(GCM_SIGNALER_SRQE, [("key", LIVE_CHAT_BASE_TANGO_KEY)])?) + .header(header::CONTENT_TYPE, "application/json+protobuf") + .body(format!(r#"[[null,null,null,[7,5],null,[["youtube_live_chat_web"],[1],[[["{}"]]]]]]"#, self.topic)) + .send() + .await? + .simd_json() + .await?; + let gsess = server_response.as_array().unwrap()[0].as_str().unwrap(); + self.gsessionid = Some(gsess.to_owned()); + Ok(()) + } + + pub async fn init_session(&mut self) -> Result<(), YouTubeError> { + let mut ofs_parameters = HashMap::new(); + ofs_parameters.insert("count", "1".to_string()); + ofs_parameters.insert("ofs", "0".to_string()); + ofs_parameters.insert( + "req0___data__", + format!(r#"[[["1",[null,null,null,[7,5],null,[["youtube_live_chat_web"],[1],[[["{}"]]]],null,null,1],null,3]]]"#, self.topic) + ); + self.session_n = 1; + let ofs = super::get_http_client() + .post(Url::parse_with_params( + GCM_SIGNALER_PSUB, + [ + ("VER", "8"), + ("gsessionid", self.gsessionid.as_ref().unwrap()), + ("key", LIVE_CHAT_BASE_TANGO_KEY), + ("RID", &self.rid.to_string()), + ("AID", &self.aid.to_string()), + ("CVER", "22"), + ("zx", Self::gen_zx().as_ref()), + ("t", "1") + ] + )?) + .header("X-WebChannel-Content-Type", "application/json+protobuf") + .form(&ofs_parameters) + .send() + .await?; + + let mut ofs_res_line = ofs.bytes().await?.lines().nth(1).unwrap().unwrap(); + let value: OwnedValue = unsafe { simd_json::from_str(&mut ofs_res_line) }?; + let value = value.as_array().unwrap()[0].as_array().unwrap(); + assert_eq!(value[0].as_usize().unwrap(), 0); + let sid = value[1].as_array().unwrap()[1].as_str().unwrap(); + self.sid = Some(sid.to_owned()); + Ok(()) + } + + pub async fn get_session_stream(&self) -> Result { + Ok(super::get_http_client() + .get(Url::parse_with_params( + GCM_SIGNALER_PSUB, + [ + ("VER", "8"), + ("gsessionid", self.gsessionid.as_ref().unwrap()), + ("key", LIVE_CHAT_BASE_TANGO_KEY), + ("RID", "rpc"), + ("SID", self.sid.as_ref().unwrap()), + ("AID", &self.aid.to_string()), + ("CI", "0"), + ("TYPE", "xmlhttp"), + ("zx", &Self::gen_zx()), + ("t", "1") + ] + )?) + .header(header::CONNECTION, "keep-alive") + .send() + .await?) + } +} + +#[derive(Debug)] +pub struct SignalerChannel { + inner: Arc> +} + +impl SignalerChannel { + pub async fn new(topic_id: impl ToString) -> Result { + Ok(SignalerChannel { + inner: Arc::new(Mutex::new(SignalerChannelInner::with_topic(topic_id))) + }) + } + + pub async fn new_from_cont(cont: &GetLiveChatResponse) -> Result { + Ok(SignalerChannel { + inner: Arc::new(Mutex::new(SignalerChannelInner::with_topic( + &cont.continuation_contents.as_ref().unwrap().live_chat_continuation.continuations[0] + .invalidation_continuation_data + .as_ref() + .unwrap() + .invalidation_id + .topic + ))) + }) + } + + pub async fn refresh_topic(&self, topic: impl ToString) { + self.inner.lock().await.topic = topic.to_string(); + } + + pub async fn spawn_event_subscriber(&self) -> Result<(broadcast::Receiver<()>, JoinHandle<()>), YouTubeError> { + let inner = Arc::clone(&self.inner); + { + let mut lock = inner.lock().await; + lock.choose_server().await?; + lock.init_session().await?; + } + let (sender, receiver) = broadcast::channel(128); + let handle = tokio::spawn(async move { + 'i: loop { + let mut req = { + let mut lock = inner.lock().await; + let _ = sender.send(()); + lock.reset(); + lock.choose_server().await.unwrap(); + lock.init_session().await.unwrap(); + lock.get_session_stream().await.unwrap() + }; + loop { + match req.chunk().await { + Ok(None) => break, + Ok(Some(s)) => { + let mut ofs_res_line = s.lines().nth(1).unwrap().unwrap(); + if let Ok(s) = unsafe { simd_json::from_str::(ofs_res_line.as_mut()) } { + let a = s.as_array().unwrap(); + { + inner.lock().await.aid = a[a.len() - 1].as_array().unwrap()[0].as_usize().unwrap(); + } + } + + if sender.send(()).is_err() { + break 'i; + } + } + Err(e) => { + eprintln!("{e:?}"); + break; + } + } + } + } + }); + Ok((receiver, handle)) + } +} diff --git a/src/youtube/types.rs b/src/youtube/types.rs index 182b1b8..9458493 100644 --- a/src/youtube/types.rs +++ b/src/youtube/types.rs @@ -124,6 +124,21 @@ pub enum MessageRun { } } +impl ToString for MessageRun { + fn to_string(&self) -> String { + match self { + Self::MessageText { text } => text.to_owned(), + Self::MessageEmoji { emoji, .. } => { + if let Some(true) = emoji.is_custom_emoji { + format!(":{}:", emoji.image.accessibility.accessibility_data.label) + } else { + emoji.image.accessibility.accessibility_data.label.to_owned() + } + } + } + } +} + #[derive(Deserialize, Debug, Clone)] #[serde(rename_all = "camelCase")] pub struct Emoji { From 9fc2981919dfc956e51b19332f9720c01a7d4dc7 Mon Sep 17 00:00:00 2001 From: "Carson M." Date: Wed, 7 Feb 2024 19:34:33 -0600 Subject: [PATCH 6/9] doki broki --- Cargo.toml | 1 + examples/youtube.rs | 80 +++++++++++++++++++++++++++------------------ src/youtube/mod.rs | 71 +++++++++++++++++++++++++++++++++------- 3 files changed, 109 insertions(+), 43 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 2f72869..31d3543 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ simd-json = { version = "0.13", optional = true } url = { version = "2.5", optional = true } rand = { version = "0.8", optional = true } regex = { version = "1.10", optional = true } +async-stream = "0.3" [dev-dependencies] anyhow = "1.0" diff --git a/examples/youtube.rs b/examples/youtube.rs index 805abba..8ae6d0f 100644 --- a/examples/youtube.rs +++ b/examples/youtube.rs @@ -1,48 +1,64 @@ -use std::future::IntoFuture; +use std::{future::IntoFuture, time::Duration}; use brainrot::youtube::{self, YouTubeChatPageProcessor}; +use tokio::time::sleep; #[tokio::main] async fn main() -> anyhow::Result<()> { - let (options, cont) = youtube::get_options_from_live_page("S144F6Cifyc").await?; + let (options, cont) = youtube::get_options_from_live_page("J2YmJL0PX5M").await?; let initial_chat = youtube::fetch_yt_chat_page(&options, &cont).await?; - let topic = initial_chat.continuation_contents.as_ref().unwrap().live_chat_continuation.continuations[0] + if let Some(invalidation_continuation) = initial_chat.continuation_contents.as_ref().unwrap().live_chat_continuation.continuations[0] .invalidation_continuation_data .as_ref() - .unwrap() - .invalidation_id - .topic - .to_owned(); - let subscriber = youtube::SignalerChannel::new(topic).await?; - let (mut receiver, _handle) = subscriber.spawn_event_subscriber().await?; - tokio::spawn(async move { - let mut processor = YouTubeChatPageProcessor::new(initial_chat, &options).unwrap(); - for msg in &processor { - println!("{}: {}", msg.author.display_name, msg.runs.iter().map(|c| c.to_string()).collect::()); - } + { + let topic = invalidation_continuation.invalidation_id.topic.to_owned(); + let subscriber = youtube::SignalerChannel::new(topic).await?; + let (mut receiver, _handle) = subscriber.spawn_event_subscriber().await?; + tokio::spawn(async move { + let mut processor = YouTubeChatPageProcessor::new(initial_chat, &options).unwrap(); + for msg in &processor { + println!("{}: {}", msg.author.display_name, msg.runs.iter().map(|c| c.to_string()).collect::()); + } - while receiver.recv().await.is_ok() { - match processor.cont().await { - Some(Ok(s)) => { - processor = s; - for msg in &processor { - println!("{}: {}", msg.author.display_name, msg.runs.iter().map(|c| c.to_string()).collect::()); - } + while receiver.recv().await.is_ok() { + match processor.cont().await { + Some(Ok(s)) => { + processor = s; + for msg in &processor { + println!("{}: {}", msg.author.display_name, msg.runs.iter().map(|c| c.to_string()).collect::()); + } - subscriber.refresh_topic(processor.signaler_topic.as_ref().unwrap()).await; - } - Some(Err(e)) => { - eprintln!("{e:?}"); - break; - } - None => { - eprintln!("none"); - break; + subscriber.refresh_topic(processor.signaler_topic.as_ref().unwrap()).await; + } + Some(Err(e)) => { + eprintln!("{e:?}"); + break; + } + None => { + eprintln!("none"); + break; + } } } + }); + _handle.into_future().await.unwrap(); + } else if let Some(timed_continuation) = initial_chat.continuation_contents.as_ref().unwrap().live_chat_continuation.continuations[0] + .timed_continuation_data + .as_ref() + { + let timeout = timed_continuation.timeout_ms as u64; + let mut processor = YouTubeChatPageProcessor::new(initial_chat, &options).unwrap(); + loop { + for msg in &processor { + println!("{}: {}", msg.author.display_name, msg.runs.iter().map(|c| c.to_string()).collect::()); + } + sleep(Duration::from_millis(timeout as _)).await; + match processor.cont().await { + Some(Ok(e)) => processor = e, + _ => break + } } - }); - _handle.into_future().await.unwrap(); + } println!("???"); Ok(()) } diff --git a/src/youtube/mod.rs b/src/youtube/mod.rs index 9aca8d9..67bf801 100644 --- a/src/youtube/mod.rs +++ b/src/youtube/mod.rs @@ -1,5 +1,6 @@ use std::{collections::VecDeque, sync::OnceLock}; +use futures_util::{Stream, StreamExt}; use regex::Regex; use reqwest::{ header::{self, HeaderMap, HeaderValue}, @@ -154,15 +155,17 @@ unsafe impl<'r> Send for YouTubeChatPageProcessor<'r> {} impl<'r> YouTubeChatPageProcessor<'r> { pub fn new(response: GetLiveChatResponse, request_options: &'r RequestOptions) -> Result { let continuation_token = if request_options.live_status { - response + let continuation = &response .continuation_contents .as_ref() .ok_or(YouTubeError::MissingContinuationContents)? .live_chat_continuation - .continuations[0] + .continuations[0]; + continuation .invalidation_continuation_data .as_ref() .map(|x| x.continuation.to_owned()) + .or_else(|| continuation.timed_continuation_data.as_ref().map(|x| x.continuation.to_owned())) } else { response .continuation_contents @@ -175,15 +178,10 @@ impl<'r> YouTubeChatPageProcessor<'r> { .map(|x| x.continuation.to_owned()) }; let signaler_topic = if request_options.live_status { - Some( - response.continuation_contents.as_ref().unwrap().live_chat_continuation.continuations[0] - .invalidation_continuation_data - .as_ref() - .unwrap() - .invalidation_id - .topic - .to_owned() - ) + response.continuation_contents.as_ref().unwrap().live_chat_continuation.continuations[0] + .invalidation_continuation_data + .as_ref() + .map(|c| c.invalidation_id.topic.to_owned()) } else { None }; @@ -297,6 +295,7 @@ impl<'r> Iterator for &YouTubeChatPageProcessor<'r> { pub async fn fetch_yt_chat_page(options: &RequestOptions, continuation: impl AsRef) -> Result { let body = GetLiveChatBody::new(continuation.as_ref(), &options.client_version, "WEB"); + println!("{}", simd_json::to_string(&body)?); let response: GetLiveChatResponse = get_http_client() .post(Url::parse_with_params( if options.live_status { TANGO_LIVE_ENDPOINT } else { TANGO_REPLAY_ENDPOINT }, @@ -307,5 +306,55 @@ pub async fn fetch_yt_chat_page(options: &RequestOptions, continuation: impl AsR .await? .simd_json() .await?; + println!( + "{}", + Url::parse_with_params( + if options.live_status { TANGO_LIVE_ENDPOINT } else { TANGO_REPLAY_ENDPOINT }, + [("key", options.api_key.as_str()), ("prettyPrint", "false")] + )? + ); Ok(response) } + +pub async fn stream( + options: &RequestOptions, + continuation: impl AsRef +) -> Result> + '_, YouTubeError> { + let initial_chat = fetch_yt_chat_page(options, continuation).await?; + let topic = initial_chat.continuation_contents.as_ref().unwrap().live_chat_continuation.continuations[0] + .invalidation_continuation_data + .as_ref() + .unwrap() + .invalidation_id + .topic + .to_owned(); + let subscriber = SignalerChannel::new(topic).await?; + let (mut receiver, _handle) = subscriber.spawn_event_subscriber().await?; + Ok(async_stream::try_stream! { + let mut processor = YouTubeChatPageProcessor::new(initial_chat, options).unwrap(); + for msg in &processor { + yield msg; + } + + while receiver.recv().await.is_ok() { + match processor.cont().await { + Some(Ok(s)) => { + processor = s; + for msg in &processor { + yield msg; + } + + subscriber.refresh_topic(processor.signaler_topic.as_ref().unwrap()).await; + } + Some(Err(e)) => { + eprintln!("{e:?}"); + break; + } + None => { + eprintln!("none"); + break; + } + } + } + }) +} From 8927b82c3f8e473968ced9437a9ff16302bdf58e Mon Sep 17 00:00:00 2001 From: "Carson M." Date: Thu, 8 Feb 2024 22:02:04 -0600 Subject: [PATCH 7/9] restructure youtube receiver --- Cargo.toml | 2 +- README.md | 25 +- examples/twitch.rs | 14 + examples/youtube.rs | 87 ++--- src/lib.rs | 14 + src/twitch/event.rs | 14 + src/twitch/identity.rs | 14 + src/twitch/mod.rs | 14 + src/util.rs | 14 + src/youtube/context.rs | 282 +++++++++++++++ src/youtube/error.rs | 66 ++++ src/youtube/mod.rs | 543 ++++++++++++++--------------- src/youtube/signaler.rs | 142 +++----- src/youtube/types.rs | 334 ------------------ src/youtube/types/get_live_chat.rs | 260 ++++++++++++++ src/youtube/types/mod.rs | 106 ++++++ src/youtube/types/streams_page.rs | 131 +++++++ src/youtube/util.rs | 24 +- 18 files changed, 1310 insertions(+), 776 deletions(-) create mode 100644 src/youtube/context.rs create mode 100644 src/youtube/error.rs delete mode 100644 src/youtube/types.rs create mode 100644 src/youtube/types/get_live_chat.rs create mode 100644 src/youtube/types/mod.rs create mode 100644 src/youtube/types/streams_page.rs diff --git a/Cargo.toml b/Cargo.toml index 31d3543..ca6f335 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,7 @@ rust-version = "1.75" [dependencies] irc = { version = "0.15", optional = true, default-features = false } -tokio = { version = "1.0", default-features = false, features = [ "net" ] } +tokio = { version = "1", default-features = false, features = [ "net" ] } futures-util = { version = "0.3", default-features = false } thiserror = "1.0" chrono = { version = "0.4", default-features = false, features = [ "clock", "std" ] } diff --git a/README.md b/README.md index 1e27516..38962ad 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,21 @@ # `brainrot` A live chat interface for Twitch & YouTube written in Rust. +## Features +- **Twitch** + * ⚡ Live IRC + * 🔓 No authentication required +- **YouTube** + * 🏆 Receive chats in real time - first library to do so + * ⚡ Low latency + * ⏪ Supports VODs + * 🔓 No authentication required + ## Usage -See [`examples/main.rs`](https://github.com/vitri-ent/brainrot/blob/examples/main.rs). +See [`examples/twitch.rs`](https://github.com/vitri-ent/brainrot/blob/examples/twitch.rs) & [`examples/youtube.rs`](https://github.com/vitri-ent/brainrot/blob/examples/youtube.rs). ```shell -$ cargo run --example main -- sinder +$ cargo run --example twitch -- sinder Spartan_N1ck: Very Generous luisfelipee23: GIGACHAD wifi882: GIGACHAD @@ -15,4 +25,15 @@ buddy_boy_joe: @sharkboticus ah LOL fair enough sinder6Laugh sinder6Laugh sinder KateRosaline14: Merry Christmas ThrillGamer2002: FirstTimeChatter ... + +$ cargo run --example youtube -- "@FUWAMOCOch" +Konami Code: makes sense +Wicho4568🐾: thank you biboo +retro: Lol +GLC H 🪐: Thanks Biboo? :face-blue-smiling::FUWAhm: +Ar5eN Vines: lol +Jic: HAHAHA +Rukh 397: :FUWAhm: +PaakType: :FUWApat::MOCOpat::FUWApat::MOCOpat: +... ``` diff --git a/examples/twitch.rs b/examples/twitch.rs index 4608d2f..acfa89f 100644 --- a/examples/twitch.rs +++ b/examples/twitch.rs @@ -1,3 +1,17 @@ +// Copyright 2024 pyke.io +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + use std::env::args; use brainrot::{twitch, TwitchChat, TwitchChatEvent}; diff --git a/examples/youtube.rs b/examples/youtube.rs index 8ae6d0f..1a3b7d2 100644 --- a/examples/youtube.rs +++ b/examples/youtube.rs @@ -1,64 +1,39 @@ -use std::{future::IntoFuture, time::Duration}; +// Copyright 2024 pyke.io +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. -use brainrot::youtube::{self, YouTubeChatPageProcessor}; -use tokio::time::sleep; +use std::env::args; + +use brainrot::youtube::{self, Action, ChatItem}; +use futures_util::StreamExt; #[tokio::main] async fn main() -> anyhow::Result<()> { - let (options, cont) = youtube::get_options_from_live_page("J2YmJL0PX5M").await?; - let initial_chat = youtube::fetch_yt_chat_page(&options, &cont).await?; - if let Some(invalidation_continuation) = initial_chat.continuation_contents.as_ref().unwrap().live_chat_continuation.continuations[0] - .invalidation_continuation_data - .as_ref() - { - let topic = invalidation_continuation.invalidation_id.topic.to_owned(); - let subscriber = youtube::SignalerChannel::new(topic).await?; - let (mut receiver, _handle) = subscriber.spawn_event_subscriber().await?; - tokio::spawn(async move { - let mut processor = YouTubeChatPageProcessor::new(initial_chat, &options).unwrap(); - for msg in &processor { - println!("{}: {}", msg.author.display_name, msg.runs.iter().map(|c| c.to_string()).collect::()); - } - - while receiver.recv().await.is_ok() { - match processor.cont().await { - Some(Ok(s)) => { - processor = s; - for msg in &processor { - println!("{}: {}", msg.author.display_name, msg.runs.iter().map(|c| c.to_string()).collect::()); - } - - subscriber.refresh_topic(processor.signaler_topic.as_ref().unwrap()).await; - } - Some(Err(e)) => { - eprintln!("{e:?}"); - break; - } - None => { - eprintln!("none"); - break; - } - } - } - }); - _handle.into_future().await.unwrap(); - } else if let Some(timed_continuation) = initial_chat.continuation_contents.as_ref().unwrap().live_chat_continuation.continuations[0] - .timed_continuation_data - .as_ref() - { - let timeout = timed_continuation.timeout_ms as u64; - let mut processor = YouTubeChatPageProcessor::new(initial_chat, &options).unwrap(); - loop { - for msg in &processor { - println!("{}: {}", msg.author.display_name, msg.runs.iter().map(|c| c.to_string()).collect::()); - } - sleep(Duration::from_millis(timeout as _)).await; - match processor.cont().await { - Some(Ok(e)) => processor = e, - _ => break - } + let context = + youtube::ChatContext::new_from_channel(args().nth(1).as_deref().unwrap_or("@miyukiwei"), youtube::ChannelSearchOptions::LatestLiveOrUpcoming).await?; + let mut stream = youtube::stream(&context).await?; + while let Some(Ok(c)) = stream.next().await { + if let Action::AddChatItem { + item: ChatItem::TextMessage { message_renderer_base, message }, + .. + } = c + { + println!( + "{}: {}", + message_renderer_base.author_name.unwrap().simple_text, + message.unwrap().runs.into_iter().map(|c| c.to_chat_string()).collect::() + ); } } - println!("???"); Ok(()) } diff --git a/src/lib.rs b/src/lib.rs index 081b870..801f754 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,17 @@ +// Copyright 2024 pyke.io +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + #[cfg(feature = "twitch")] pub mod twitch; #[cfg(feature = "twitch")] diff --git a/src/twitch/event.rs b/src/twitch/event.rs index eb67c2c..82900af 100644 --- a/src/twitch/event.rs +++ b/src/twitch/event.rs @@ -1,3 +1,17 @@ +// Copyright 2024 pyke.io +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + use std::{ collections::HashMap, num::{NonZeroU16, NonZeroU32} diff --git a/src/twitch/identity.rs b/src/twitch/identity.rs index 74767e8..cc29c24 100644 --- a/src/twitch/identity.rs +++ b/src/twitch/identity.rs @@ -1,3 +1,17 @@ +// Copyright 2024 pyke.io +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + /// Represents a type that can be used to identify the client. pub trait TwitchIdentity { /// Converts this type into a tuple of `(username, Option)`. diff --git a/src/twitch/mod.rs b/src/twitch/mod.rs index d93bfda..86b3ed2 100644 --- a/src/twitch/mod.rs +++ b/src/twitch/mod.rs @@ -1,3 +1,17 @@ +// Copyright 2024 pyke.io +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + use std::{ pin::Pin, task::{Context, Poll} diff --git a/src/util.rs b/src/util.rs index ed16f3c..09670e6 100644 --- a/src/util.rs +++ b/src/util.rs @@ -1,3 +1,17 @@ +// Copyright 2024 pyke.io +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + pub trait MapNonempty { type T; diff --git a/src/youtube/context.rs b/src/youtube/context.rs new file mode 100644 index 0000000..0f5878f --- /dev/null +++ b/src/youtube/context.rs @@ -0,0 +1,282 @@ +// Copyright 2024 pyke.io +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::OnceLock; + +use regex::Regex; +use url::Url; + +use super::{ + get_http_client, + types::streams_page::{ + FeedContentsRenderer, PageContentsRenderer, RichGridItem, RichItemContent, TabItemRenderer, ThumbnailOverlay, VideoTimeStatus, YouTubeInitialData + }, + Error +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LiveStreamStatus { + Upcoming, + Live, + Replay +} + +impl LiveStreamStatus { + #[inline] + pub fn updates_live(&self) -> bool { + matches!(self, LiveStreamStatus::Upcoming | LiveStreamStatus::Live) + } +} + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub enum ChannelSearchOptions { + /// Get the live chat of the latest live stream, or the pre-stream chat of the latest upcoming stream if no stream + /// is currently live. + LatestLiveOrUpcoming, + /// Get the live chat of the first live stream, or the pre-stream chat of the first upcoming stream if no stream + /// is currently live. + #[default] + FirstLiveOrUpcoming, + /// Get the live chat of the first live stream. + FirstLive, + /// Get the live chat of the latest live stream. + LatestLive +} + +#[derive(Clone, Debug)] +pub struct ChatContext { + pub(crate) id: String, + pub(crate) api_key: String, + pub(crate) client_version: String, + pub(crate) initial_continuation: String, + pub(crate) tango_api_key: Option, + pub(crate) live_status: LiveStreamStatus +} + +impl ChatContext { + pub async fn new_from_channel(channel_id: impl AsRef, options: ChannelSearchOptions) -> Result { + let channel_id = channel_id.as_ref(); + let channel_id = if channel_id.starts_with("UC") || channel_id.starts_with('@') { + channel_id + } else { + Self::parse_channel_link(channel_id).ok_or_else(|| Error::InvalidChannelID(channel_id.to_string()))? + }; + let page_contents = get_http_client() + .get(if channel_id.starts_with('@') { + format!("https://www.youtube.com/{channel_id}/streams") + } else { + format!("https://www.youtube.com/channel/{channel_id}/streams") + }) + .send() + .await? + .text() + .await?; + + static YT_INITIAL_DATA_REGEX: OnceLock = OnceLock::new(); + let yt_initial_data: YouTubeInitialData = unsafe { + simd_json::from_str( + &mut YT_INITIAL_DATA_REGEX + .get_or_init(|| Regex::new(r#"var ytInitialData\s*=\s*(\{.+?\});"#).unwrap()) + .captures(&page_contents) + .ok_or_else(|| Error::NoChatContinuation)? + .get(1) + .ok_or(Error::MissingInitialData)? + .as_str() + .to_owned() + ) + }?; + + let mut live_id = None; + match yt_initial_data.contents { + PageContentsRenderer::TwoColumnBrowseResultsRenderer { tabs } => match tabs + .iter() + .find(|c| match c { + TabItemRenderer::TabRenderer { title, content, .. } => content.is_some() && title == "Live", + TabItemRenderer::ExpandableTabRenderer { .. } => false + }) + .ok_or_else(|| Error::NoMatchingStream(channel_id.to_string()))? + { + TabItemRenderer::TabRenderer { content, .. } => match content.as_ref().unwrap() { + FeedContentsRenderer::RichGridRenderer { contents, .. } => { + let finder = |c: &&RichGridItem| match c { + RichGridItem::RichItemRenderer { content, .. } => match content { + RichItemContent::VideoRenderer { thumbnail_overlays, video_id, .. } => thumbnail_overlays.iter().any(|c| match c { + ThumbnailOverlay::TimeStatus { style, .. } => { + if *style == VideoTimeStatus::Live { + live_id = Some((video_id.to_owned(), true)); + true + } else { + if *style == VideoTimeStatus::Upcoming + && matches!(options, ChannelSearchOptions::FirstLiveOrUpcoming | ChannelSearchOptions::LatestLiveOrUpcoming) + { + match &live_id { + None => { + live_id = Some((video_id.to_owned(), false)); + } + Some((_, false)) => { + live_id = Some((video_id.to_owned(), false)); + } + Some((_, true)) => {} + } + } + false + } + } + _ => false + }) + }, + RichGridItem::ContinuationItemRenderer { .. } => false + }; + if matches!(options, ChannelSearchOptions::FirstLive | ChannelSearchOptions::FirstLiveOrUpcoming) { + contents.iter().rev().find(finder) + } else { + contents.iter().find(finder) + } + .ok_or_else(|| Error::NoMatchingStream(channel_id.to_string()))? + } + FeedContentsRenderer::SectionListRenderer { .. } => return Err(Error::NoMatchingStream(channel_id.to_string())) + }, + TabItemRenderer::ExpandableTabRenderer { .. } => unreachable!() + } + }; + + ChatContext::new_from_live(live_id.ok_or_else(|| Error::NoMatchingStream(channel_id.to_string()))?.0).await + } + + pub async fn new_from_live(id: impl AsRef) -> Result { + let id = id.as_ref(); + let live_id = if id.is_ascii() && id.len() == 11 { + id + } else { + Self::parse_stream_link(id).ok_or_else(|| Error::InvalidVideoID(id.to_string()))? + }; + let page_contents = get_http_client() + .get(format!("https://www.youtube.com/watch?v={live_id}")) + .send() + .await? + .text() + .await?; + + static LIVE_STREAM_REGEX: OnceLock = OnceLock::new(); + let live_status = if LIVE_STREAM_REGEX + .get_or_init(|| Regex::new(r#"['"]isLiveContent['"]:\s*(true)"#).unwrap()) + .find(&page_contents) + .is_some() + { + static LIVE_NOW_REGEX: OnceLock = OnceLock::new(); + static REPLAY_REGEX: OnceLock = OnceLock::new(); + if LIVE_NOW_REGEX + .get_or_init(|| Regex::new(r#"['"]isLiveNow['"]:\s*(true)"#).unwrap()) + .find(&page_contents) + .is_some() + { + LiveStreamStatus::Live + } else if REPLAY_REGEX + .get_or_init(|| Regex::new(r#"['"]isReplay['"]:\s*(true)"#).unwrap()) + .find(&page_contents) + .is_some() + { + LiveStreamStatus::Replay + } else { + LiveStreamStatus::Upcoming + } + } else { + return Err(Error::NotStream(live_id.to_string())); + }; + + static INNERTUBE_API_KEY_REGEX: OnceLock = OnceLock::new(); + let api_key = match INNERTUBE_API_KEY_REGEX + .get_or_init(|| Regex::new(r#"['"]INNERTUBE_API_KEY['"]:\s*['"](.+?)['"]"#).unwrap()) + .captures(&page_contents) + .and_then(|captures| captures.get(1)) + { + Some(matched) => matched.as_str().to_string(), + None => return Err(Error::NoInnerTubeKey) + }; + + static TANGO_API_KEY_REGEX: OnceLock = OnceLock::new(); + let tango_api_key = TANGO_API_KEY_REGEX + .get_or_init(|| Regex::new(r#"['"]LIVE_CHAT_BASE_TANGO_CONFIG['"]:\s*\{\s*['"]apiKey['"]\s*:\s*['"](.+?)['"]"#).unwrap()) + .captures(&page_contents) + .and_then(|captures| captures.get(1).map(|c| c.as_str().to_string())); + + static CLIENT_VERSION_REGEX: OnceLock = OnceLock::new(); + let client_version = match CLIENT_VERSION_REGEX + .get_or_init(|| Regex::new(r#"['"]clientVersion['"]:\s*['"]([\d.]+?)['"]"#).unwrap()) + .captures(&page_contents) + .and_then(|captures| captures.get(1)) + { + Some(matched) => matched.as_str().to_string(), + None => "2.20240207.07.00".to_string() + }; + + static LIVE_CONTINUATION_REGEX: OnceLock = OnceLock::new(); + static REPLAY_CONTINUATION_REGEX: OnceLock = OnceLock::new(); + let continuation_regex = if live_status.updates_live() { + LIVE_CONTINUATION_REGEX.get_or_init(|| Regex::new( + r#"Live chat['"],\s*['"]selected['"]:\s*(?:true|false),\s*['"]continuation['"]:\s*\{\s*['"]reloadContinuationData['"]:\s*\{['"]continuation['"]:\s*['"](.+?)['"]"# + ).unwrap()) + } else { + REPLAY_CONTINUATION_REGEX.get_or_init(|| { + Regex::new( + r#"Top chat replay['"],\s*['"]selected['"]:\s*true,\s*['"]continuation['"]:\s*\{\s*['"]reloadContinuationData['"]:\s*\{['"]continuation['"]:\s*['"](.+?)['"]"# + ) + .unwrap() + }) + }; + let continuation = match continuation_regex.captures(&page_contents).and_then(|captures| captures.get(1)) { + Some(matched) => matched.as_str().to_string(), + None => return Err(Error::NoChatContinuation) + }; + + Ok(ChatContext { + id: live_id.to_string(), + api_key, + client_version, + tango_api_key, + initial_continuation: continuation, + live_status + }) + } + + fn parse_stream_link(url: &str) -> Option<&str> { + static LINK_RE: OnceLock = OnceLock::new(); + LINK_RE + .get_or_init(|| Regex::new(r#"(?:https?:\/\/)?(?:www\.)?youtu\.?be(?:\.com)?\/?.*(?:watch|embed)?(?:.*v=|v\/|\/)([A-Za-z0-9-_]+)"#).unwrap()) + .captures(url) + .and_then(|c| c.get(1)) + .map(|c| c.as_str()) + } + + fn parse_channel_link(url: &str) -> Option<&str> { + static CHANNEL_RE: OnceLock = OnceLock::new(); + CHANNEL_RE + .get_or_init(|| Regex::new(r#"^(?:https?:\/\/)?(?:www\.)?youtube\.com\/(?:channel\/(UC[\w-]{21}[AQgw])|(@[\w]+))$"#).unwrap()) + .captures(url) + .and_then(|c| c.get(1)) + .map(|c| c.as_str()) + } + + pub fn id(&self) -> &str { + &self.id + } + + pub fn url(&self) -> Url { + Url::parse(&format!("https://www.youtube.com/watch?v={}", self.id)).unwrap() + } + + pub fn status(&self) -> LiveStreamStatus { + self.live_status + } +} diff --git a/src/youtube/error.rs b/src/youtube/error.rs new file mode 100644 index 0000000..dbfb3e2 --- /dev/null +++ b/src/youtube/error.rs @@ -0,0 +1,66 @@ +// Copyright 2024 pyke.io +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use reqwest::StatusCode; +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum Error { + #[error("Invalid YouTube video ID or URL: {0}")] + InvalidVideoID(String), + #[error("Invalid YouTube channel ID or URL: {0}")] + InvalidChannelID(String), + #[error("Channel {0} has no live stream matching the options criteria")] + NoMatchingStream(String), + #[error("Missing `ytInitialData` structure from channel streams page.")] + MissingInitialData, + #[error("error when deserializing: {0}")] + Deserialization(#[from] simd_json::Error), + #[error("missing continuation contents")] + MissingContinuationContents, + #[error("reached end of continuation")] + EndOfContinuation, + #[error("request timed out")] + TimedOut, + #[error("request returned bad HTTP status: {0}")] + BadStatus(StatusCode), + #[error("request error: {0}")] + GeneralRequest(reqwest::Error), + #[error("{0} is not a live stream")] + NotStream(String), + #[error("Failed to match InnerTube API key")] + NoInnerTubeKey, + #[error("Chat continuation token could not be found.")] + NoChatContinuation, + #[error("Error parsing URL: {0}")] + URLParseError(#[from] url::ParseError) +} + +impl Error { + pub fn is_fatal(&self) -> bool { + !matches!(self, Error::TimedOut) + } +} + +impl From for Error { + fn from(value: reqwest::Error) -> Self { + if value.is_timeout() { + Error::TimedOut + } else if value.is_status() { + Error::BadStatus(value.status().unwrap()) + } else { + Error::GeneralRequest(value) + } + } +} diff --git a/src/youtube/mod.rs b/src/youtube/mod.rs index 67bf801..a8a488b 100644 --- a/src/youtube/mod.rs +++ b/src/youtube/mod.rs @@ -1,218 +1,119 @@ -use std::{collections::VecDeque, sync::OnceLock}; +// Copyright 2024 pyke.io +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. -use futures_util::{Stream, StreamExt}; -use regex::Regex; -use reqwest::{ - header::{self, HeaderMap, HeaderValue}, - StatusCode -}; +use std::{collections::HashSet, io::BufRead, pin::Pin, sync::OnceLock, time::Duration}; + +use futures_util::Stream; +use reqwest::header::{self, HeaderMap, HeaderValue}; +use simd_json::base::{ValueAsContainer, ValueAsScalar}; use thiserror::Error; -use tokio::sync::Mutex; -use url::Url; +use tokio::time::sleep; +mod context; +mod error; mod signaler; mod types; mod util; -pub use self::signaler::SignalerChannel; + +pub use self::{ + context::{ChannelSearchOptions, ChatContext, LiveStreamStatus}, + error::Error, + types::{ + get_live_chat::{Action, ChatItem, MessageRendererBase}, + ImageContainer, LocalizedRun, LocalizedText, Thumbnail, UnlocalizedText + } +}; use self::{ - types::{Action, GetLiveChatBody, GetLiveChatResponse, MessageRun}, - util::{SimdJsonRequestBody, SimdJsonResponseBody} + signaler::SignalerChannelInner, + types::get_live_chat::{Continuation, GetLiveChatResponse} }; const TANGO_LIVE_ENDPOINT: &str = "https://www.youtube.com/youtubei/v1/live_chat/get_live_chat"; const TANGO_REPLAY_ENDPOINT: &str = "https://www.youtube.com/youtubei/v1/live_chat/get_live_chat_replay"; -const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:123.0) Gecko/20100101 Firefox/123.0"; - -#[derive(Debug, Error)] -pub enum YouTubeError { - #[error("impossible regex error")] - Regex(#[from] regex::Error), - #[error("error when deserializing: {0}")] - Deserialization(#[from] simd_json::Error), - #[error("missing continuation contents")] - MissingContinuationContents, - #[error("reached end of continuation")] - EndOfContinuation, - #[error("request timed out")] - TimedOut, - #[error("request returned bad HTTP status: {0}")] - BadStatus(StatusCode), - #[error("request error: {0}")] - GeneralRequest(reqwest::Error), - #[error("{0} is not a live stream")] - NotStream(String), - #[error("Failed to match InnerTube API key")] - NoInnerTubeKey, - #[error("Chat continuation token could not be found.")] - NoChatContinuation, - #[error("Error parsing URL: {0}")] - URLParseError(#[from] url::ParseError) -} - -impl From for YouTubeError { - fn from(value: reqwest::Error) -> Self { - if value.is_timeout() { - YouTubeError::TimedOut - } else if value.is_status() { - YouTubeError::BadStatus(value.status().unwrap()) - } else { - YouTubeError::GeneralRequest(value) - } - } -} - pub(crate) fn get_http_client() -> &'static reqwest::Client { static HTTP_CLIENT: OnceLock = OnceLock::new(); HTTP_CLIENT.get_or_init(|| { let mut headers = HeaderMap::new(); // Set our Accept-Language to en-US so we can properly match substrings headers.append(header::ACCEPT_LANGUAGE, HeaderValue::from_static("en-US,en;q=0.5")); - headers.append(header::USER_AGENT, HeaderValue::from_static(USER_AGENT)); + headers.append(header::USER_AGENT, HeaderValue::from_static("Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:123.0) Gecko/20100101 Firefox/123.0")); + // Referer is required by Signaler endpoints. headers.append(header::REFERER, HeaderValue::from_static("https://www.youtube.com/")); reqwest::Client::builder().default_headers(headers).build().unwrap() }) } -#[derive(Clone, Debug)] -pub struct RequestOptions { - pub(crate) api_key: String, - pub(crate) client_version: String, - pub(crate) live_status: bool -} - -pub async fn get_options_from_live_page(live_id: impl AsRef) -> Result<(RequestOptions, String), YouTubeError> { - let live_id = live_id.as_ref(); - let page_contents = get_http_client() - .get(format!("https://www.youtube.com/watch?v={live_id}")) - .send() - .await? - .text() - .await?; - - let live_status: bool; - let live_now_regex = Regex::new(r#"['"]isLiveNow['"]:\s*(true)"#)?; - let not_replay_regex = Regex::new(r#"['"]isReplay['"]:\s*(true)"#)?; - if live_now_regex.find(&page_contents).is_some() { - live_status = true; - } else if not_replay_regex.find(&page_contents).is_some() { - live_status = false; - } else { - return Err(YouTubeError::NotStream(live_id.to_string())); - } - - let api_key_regex = Regex::new(r#"['"]INNERTUBE_API_KEY['"]:\s*['"](.+?)['"]"#).unwrap(); - let api_key = match api_key_regex.captures(&page_contents).and_then(|captures| captures.get(1)) { - Some(matched) => matched.as_str().to_string(), - None => return Err(YouTubeError::NoInnerTubeKey) - }; - - let client_version_regex = Regex::new(r#"['"]clientVersion['"]:\s*['"]([\d.]+?)['"]"#).unwrap(); - let client_version = match client_version_regex.captures(&page_contents).and_then(|captures| captures.get(1)) { - Some(matched) => matched.as_str().to_string(), - None => "2.20230801.08.00".to_string() - }; - - let continuation_regex = if live_status { - Regex::new( - r#"Live chat['"],\s*['"]selected['"]:\s*(?:true|false),\s*['"]continuation['"]:\s*\{\s*['"]reloadContinuationData['"]:\s*\{['"]continuation['"]:\s*['"](.+?)['"]"# - )? - } else { - Regex::new( - r#"Top chat replay['"],\s*['"]selected['"]:\s*true,\s*['"]continuation['"]:\s*\{\s*['"]reloadContinuationData['"]:\s*\{['"]continuation['"]:\s*['"](.+?)['"]"# - )? - }; - let continuation = match continuation_regex.captures(&page_contents).and_then(|captures| captures.get(1)) { - Some(matched) => matched.as_str().to_string(), - None => return Err(YouTubeError::NoChatContinuation) - }; - - Ok((RequestOptions { api_key, client_version, live_status }, continuation)) -} -pub struct Author { - pub display_name: String, - pub id: String, - pub avatar: String -} - -pub struct ChatMessage { - pub runs: Vec, - pub is_super: bool, - pub author: Author, - pub timestamp: i64, - pub time_delta: Option -} - -pub struct YouTubeChatPageProcessor<'r> { - actions: Mutex>, - request_options: &'r RequestOptions, +struct ActionChunk<'r> { + actions: Vec, + ctx: &'r ChatContext, continuation_token: Option, - pub signaler_topic: Option + pub(crate) signaler_topic: Option } -unsafe impl<'r> Send for YouTubeChatPageProcessor<'r> {} +unsafe impl<'r> Send for ActionChunk<'r> {} -impl<'r> YouTubeChatPageProcessor<'r> { - pub fn new(response: GetLiveChatResponse, request_options: &'r RequestOptions) -> Result { - let continuation_token = if request_options.live_status { - let continuation = &response - .continuation_contents - .as_ref() - .ok_or(YouTubeError::MissingContinuationContents)? - .live_chat_continuation - .continuations[0]; - continuation - .invalidation_continuation_data - .as_ref() - .map(|x| x.continuation.to_owned()) - .or_else(|| continuation.timed_continuation_data.as_ref().map(|x| x.continuation.to_owned())) - } else { - response - .continuation_contents - .as_ref() - .ok_or(YouTubeError::MissingContinuationContents)? - .live_chat_continuation - .continuations[0] - .live_chat_replay_continuation_data - .as_ref() - .map(|x| x.continuation.to_owned()) +impl<'r> ActionChunk<'r> { + pub fn new(response: GetLiveChatResponse, ctx: &'r ChatContext) -> Result { + let continuation_token = match &response.continuation_contents.live_chat_continuation.continuations[0] { + Continuation::Invalidation { continuation, .. } => continuation.to_owned(), + Continuation::Timed { continuation, .. } => continuation.to_owned(), + Continuation::Replay { continuation, .. } => continuation.to_owned() }; - let signaler_topic = if request_options.live_status { - response.continuation_contents.as_ref().unwrap().live_chat_continuation.continuations[0] - .invalidation_continuation_data - .as_ref() - .map(|c| c.invalidation_id.topic.to_owned()) - } else { - None + let signaler_topic = match &response.continuation_contents.live_chat_continuation.continuations[0] { + Continuation::Invalidation { invalidation_id, .. } => Some(invalidation_id.topic.to_owned()), + _ => None }; Ok(Self { - actions: Mutex::new(VecDeque::from(if request_options.live_status { + actions: if ctx.live_status.updates_live() { response .continuation_contents - .ok_or(YouTubeError::MissingContinuationContents)? .live_chat_continuation .actions .unwrap_or_default() + .into_iter() + .map(|f| f.action) + .collect() } else { response .continuation_contents - .ok_or(YouTubeError::MissingContinuationContents)? .live_chat_continuation .actions - .ok_or(YouTubeError::EndOfContinuation)? - })), - request_options, - continuation_token, + .ok_or(Error::EndOfContinuation)? + .into_iter() + .flat_map(|f| match f.action { + Action::ReplayChat { actions, .. } => actions, + f => vec![f] + }) + .collect() + }, + ctx, + continuation_token: Some(continuation_token), signaler_topic }) } - async fn next_page(&self, continuation_token: &String) -> Result { - let page = fetch_yt_chat_page(self.request_options, continuation_token).await?; - YouTubeChatPageProcessor::new(page, self.request_options) + pub fn iter(&self) -> std::slice::Iter<'_, Action> { + self.actions.iter() } - pub async fn cont(&self) -> Option> { + async fn next_page(&self, continuation_token: &String) -> Result { + let page = GetLiveChatResponse::fetch(self.ctx, continuation_token).await?; + ActionChunk::new(page, self.ctx) + } + + pub async fn cont(&self) -> Option> { if let Some(continuation_token) = &self.continuation_token { Some(self.next_page(continuation_token).await) } else { @@ -221,140 +122,212 @@ impl<'r> YouTubeChatPageProcessor<'r> { } } -impl<'r> Iterator for &YouTubeChatPageProcessor<'r> { - type Item = ChatMessage; +impl<'r> IntoIterator for ActionChunk<'r> { + type Item = Action; + type IntoIter = std::vec::IntoIter; - fn next(&mut self) -> Option { - let mut next_action = None; - while next_action.is_none() { - match self.actions.try_lock().unwrap().pop_front() { - Some(action) => { - if let Some(replay) = action.replay_chat_item_action { - for action in replay.actions { - if next_action.is_some() { - break; - } + fn into_iter(self) -> Self::IntoIter { + self.actions.into_iter() + } +} - if let Some(add_chat_item_action) = action.add_chat_item_action { - if let Some(text_message_renderer) = &add_chat_item_action.item.live_chat_text_message_renderer { - if text_message_renderer.message.is_some() { - next_action.replace((add_chat_item_action, Some(replay.video_offset_time_msec))); - } - } else if let Some(superchat_renderer) = &add_chat_item_action.item.live_chat_paid_message_renderer { - if superchat_renderer.live_chat_text_message_renderer.message.is_some() { - next_action.replace((add_chat_item_action, Some(replay.video_offset_time_msec))); - } - } +pub async fn stream(options: &ChatContext) -> Result> + '_>>, Error> { + let initial_chat = GetLiveChatResponse::fetch(options, &options.initial_continuation).await?; + + let (mut yield_tx, yield_rx) = unsafe { async_stream::__private::yielder::pair() }; + + Ok(Box::pin(async_stream::__private::AsyncStream::new(yield_rx, async move { + let mut seen_messages = HashSet::new(); + + match &initial_chat.continuation_contents.live_chat_continuation.continuations[0] { + Continuation::Invalidation { invalidation_id, .. } => { + let topic = invalidation_id.topic.to_owned(); + + let mut chunk = ActionChunk::new(initial_chat, options).unwrap(); + + let mut channel = SignalerChannelInner::with_topic(topic, options.tango_api_key.as_ref().unwrap()); + channel.choose_server().await.unwrap(); + channel.init_session().await.unwrap(); + + for action in chunk.iter() { + match action { + Action::AddChatItem { item, .. } => { + if !seen_messages.contains(item.id()) { + yield_tx.send(Ok(action.to_owned())).await; + seen_messages.insert(item.id().to_owned()); } } - } else if let Some(action) = action.add_chat_item_action { - if let Some(text_message_renderer) = &action.item.live_chat_text_message_renderer { - if text_message_renderer.message.is_some() { - next_action.replace((action, None)); - } - } else if let Some(superchat_renderer) = &action.item.live_chat_paid_message_renderer { - if superchat_renderer.live_chat_text_message_renderer.message.is_some() { - next_action.replace((action, None)); + Action::ReplayChat { actions, .. } => { + for action in actions { + if let Action::AddChatItem { .. } = action { + yield_tx.send(Ok(action.to_owned())).await; + } } } + action => { + yield_tx.send(Ok(action.to_owned())).await; + } } } - None => return None - } - } - let (next_action, time_delta) = next_action.unwrap(); - let is_super = next_action.item.live_chat_paid_message_renderer.is_some(); - let renderer = if let Some(renderer) = next_action.item.live_chat_text_message_renderer { - renderer - } else if let Some(renderer) = next_action.item.live_chat_paid_message_renderer { - renderer.live_chat_text_message_renderer - } else { - unimplemented!() - }; + 'i: loop { + match chunk.cont().await { + Some(Ok(c)) => chunk = c, + Some(Err(err)) => eprintln!("{err:?}"), + _ => break 'i + }; - Some(ChatMessage { - runs: renderer.message.unwrap().runs, - is_super, - author: Author { - display_name: renderer - .message_renderer_base - .author_name - .map(|x| x.simple_text) - .unwrap_or_else(|| renderer.message_renderer_base.author_external_channel_id.to_owned()), - id: renderer.message_renderer_base.author_external_channel_id.to_owned(), - avatar: renderer.message_renderer_base.author_photo.thumbnails[renderer.message_renderer_base.author_photo.thumbnails.len() - 1] - .url - .to_owned() - }, - timestamp: renderer.message_renderer_base.timestamp_usec.timestamp_millis(), - time_delta - }) - } -} + for action in chunk.iter() { + match action { + Action::AddChatItem { item, .. } => { + if !seen_messages.contains(item.id()) { + yield_tx.send(Ok(action.to_owned())).await; + seen_messages.insert(item.id().to_owned()); + } + } + Action::ReplayChat { actions, .. } => { + for action in actions { + if let Action::AddChatItem { .. } = action { + yield_tx.send(Ok(action.to_owned())).await; + } + } + } + action => { + yield_tx.send(Ok(action.to_owned())).await; + } + } + } -pub async fn fetch_yt_chat_page(options: &RequestOptions, continuation: impl AsRef) -> Result { - let body = GetLiveChatBody::new(continuation.as_ref(), &options.client_version, "WEB"); - println!("{}", simd_json::to_string(&body)?); - let response: GetLiveChatResponse = get_http_client() - .post(Url::parse_with_params( - if options.live_status { TANGO_LIVE_ENDPOINT } else { TANGO_REPLAY_ENDPOINT }, - [("key", options.api_key.as_str()), ("prettyPrint", "false")] - )?) - .simd_json(&body)? - .send() - .await? - .simd_json() - .await?; - println!( - "{}", - Url::parse_with_params( - if options.live_status { TANGO_LIVE_ENDPOINT } else { TANGO_REPLAY_ENDPOINT }, - [("key", options.api_key.as_str()), ("prettyPrint", "false")] - )? - ); - Ok(response) -} + let mut req = { + channel.reset(); + channel.choose_server().await.unwrap(); + channel.init_session().await.unwrap(); + channel.get_session_stream().await.unwrap() + }; + loop { + match req.chunk().await { + Ok(Some(s)) => { + let mut ofs_res_line = s.lines().nth(1).unwrap().unwrap(); + if let Ok(s) = unsafe { simd_json::from_str::(ofs_res_line.as_mut()) } { + let a = s.as_array().unwrap(); + { + channel.aid = a[a.len() - 1].as_array().unwrap()[0].as_usize().unwrap(); + } + } -pub async fn stream( - options: &RequestOptions, - continuation: impl AsRef -) -> Result> + '_, YouTubeError> { - let initial_chat = fetch_yt_chat_page(options, continuation).await?; - let topic = initial_chat.continuation_contents.as_ref().unwrap().live_chat_continuation.continuations[0] - .invalidation_continuation_data - .as_ref() - .unwrap() - .invalidation_id - .topic - .to_owned(); - let subscriber = SignalerChannel::new(topic).await?; - let (mut receiver, _handle) = subscriber.spawn_event_subscriber().await?; - Ok(async_stream::try_stream! { - let mut processor = YouTubeChatPageProcessor::new(initial_chat, options).unwrap(); - for msg in &processor { - yield msg; - } + match chunk.cont().await { + Some(Ok(c)) => chunk = c, + Some(Err(err)) => eprintln!("{err:?}"), + _ => break 'i + }; + channel.topic = chunk.signaler_topic.clone().unwrap(); - while receiver.recv().await.is_ok() { - match processor.cont().await { - Some(Ok(s)) => { - processor = s; - for msg in &processor { - yield msg; + for action in chunk.iter() { + match action { + Action::AddChatItem { item, .. } => { + if !seen_messages.contains(item.id()) { + yield_tx.send(Ok(action.to_owned())).await; + seen_messages.insert(item.id().to_owned()); + } + } + Action::ReplayChat { actions, .. } => { + for action in actions { + if let Action::AddChatItem { .. } = action { + yield_tx.send(Ok(action.to_owned())).await; + } + } + } + action => { + yield_tx.send(Ok(action.to_owned())).await; + } + } + } + } + Ok(None) => break, + Err(e) => { + eprintln!("{e:?}"); + break; + } + } } - subscriber.refresh_topic(processor.signaler_topic.as_ref().unwrap()).await; + seen_messages.clear(); } - Some(Err(e)) => { - eprintln!("{e:?}"); - break; + } + Continuation::Replay { .. } => { + let chunk = ActionChunk::new(initial_chat, options).unwrap(); + for action in chunk.iter() { + match action { + Action::AddChatItem { .. } => { + yield_tx.send(Ok(action.to_owned())).await; + } + Action::ReplayChat { actions, .. } => { + for action in actions { + if let Action::AddChatItem { .. } = action { + yield_tx.send(Ok(action.to_owned())).await; + } + } + } + action => { + yield_tx.send(Ok(action.to_owned())).await; + } + } } - None => { - eprintln!("none"); - break; + + while let Some(Ok(chunk)) = chunk.cont().await { + for action in chunk.iter() { + match action { + Action::AddChatItem { item, .. } => { + if !seen_messages.contains(item.id()) { + yield_tx.send(Ok(action.to_owned())).await; + seen_messages.insert(item.id().to_owned()); + } + } + Action::ReplayChat { actions, .. } => { + for action in actions { + if let Action::AddChatItem { .. } = action { + yield_tx.send(Ok(action.to_owned())).await; + } + } + } + action => { + yield_tx.send(Ok(action.to_owned())).await; + } + } + } + } + } + Continuation::Timed { timeout_ms, .. } => { + let timeout = Duration::from_millis(*timeout_ms as _); + let mut chunk = ActionChunk::new(initial_chat, options).unwrap(); + loop { + for action in chunk.iter() { + match action { + Action::AddChatItem { item, .. } => { + if !seen_messages.contains(item.id()) { + yield_tx.send(Ok(action.to_owned())).await; + seen_messages.insert(item.id().to_owned()); + } + } + Action::ReplayChat { actions, .. } => { + for action in actions { + if let Action::AddChatItem { .. } = action { + yield_tx.send(Ok(action.to_owned())).await; + } + } + } + action => { + yield_tx.send(Ok(action.to_owned())).await; + } + } + } + sleep(timeout).await; + match chunk.cont().await { + Some(Ok(e)) => chunk = e, + _ => break + } } } } - }) + }))) } diff --git a/src/youtube/signaler.rs b/src/youtube/signaler.rs index e89f45e..3c686f8 100644 --- a/src/youtube/signaler.rs +++ b/src/youtube/signaler.rs @@ -1,4 +1,35 @@ -use std::{collections::HashMap, io::BufRead, iter, sync::Arc}; +// Copyright 2024 pyke.io +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// /////////////////////////////////////////////// // +// In the realm of YouTube's code, beware, // +// Where chaos reigns, and clarity's rare. // +// Thirty hours deep, I dove with despair, // +// Into the labyrinth of the Signaler lair. // +// // +// Oh, the trials faced, the struggles endured, // +// As I wrestled with quirks, my sanity blurred. // +// In the murky depths of DevTools' lair, // +// I found myself lost, tangled in despair. // +// // +// So heed this warning, brave coder, take care, // +// As you venture forth, through Signaler's snare. // +// My code may be messy, a sight to beware, // +// But through the chaos, a solution may glare. // +// /////////////////////////////////////////////// // + +use std::{collections::HashMap, io::BufRead, iter}; use rand::Rng; use reqwest::{header, Response}; @@ -6,33 +37,29 @@ use simd_json::{ base::{ValueAsContainer, ValueAsScalar}, OwnedValue }; -use tokio::{ - sync::{broadcast, Mutex}, - task::JoinHandle -}; use url::Url; -use super::{types::GetLiveChatResponse, util::SimdJsonResponseBody, YouTubeError}; +use super::{util::SimdJsonResponseBody, Error}; const GCM_SIGNALER_SRQE: &str = "https://signaler-pa.youtube.com/punctual/v1/chooseServer"; const GCM_SIGNALER_PSUB: &str = "https://signaler-pa.youtube.com/punctual/multi-watch/channel"; -const LIVE_CHAT_BASE_TANGO_KEY: &str = "AIzaSyDZNkyC-AtROwMBpLfevIvqYk-Gfi8ZOeo"; - #[derive(Debug, Default)] -struct SignalerChannelInner { - topic: String, +pub struct SignalerChannelInner { + pub(crate) topic: String, + tango_key: String, gsessionid: Option, sid: Option, rid: usize, - aid: usize, + pub(crate) aid: usize, session_n: usize } impl SignalerChannelInner { - pub fn with_topic(topic: impl ToString) -> Self { + pub fn with_topic(topic: impl ToString, tango_key: impl ToString) -> Self { Self { topic: topic.to_string(), + tango_key: tango_key.to_string(), ..Default::default() } } @@ -51,9 +78,9 @@ impl SignalerChannelInner { iter::repeat_with(|| CHARSET[rng.gen_range(0..CHARSET.len())] as char).take(11).collect() } - pub async fn choose_server(&mut self) -> Result<(), YouTubeError> { + pub async fn choose_server(&mut self) -> Result<(), Error> { let server_response: OwnedValue = super::get_http_client() - .post(Url::parse_with_params(GCM_SIGNALER_SRQE, [("key", LIVE_CHAT_BASE_TANGO_KEY)])?) + .post(Url::parse_with_params(GCM_SIGNALER_SRQE, [("key", &self.tango_key)])?) .header(header::CONTENT_TYPE, "application/json+protobuf") .body(format!(r#"[[null,null,null,[7,5],null,[["youtube_live_chat_web"],[1],[[["{}"]]]]]]"#, self.topic)) .send() @@ -65,7 +92,7 @@ impl SignalerChannelInner { Ok(()) } - pub async fn init_session(&mut self) -> Result<(), YouTubeError> { + pub async fn init_session(&mut self) -> Result<(), Error> { let mut ofs_parameters = HashMap::new(); ofs_parameters.insert("count", "1".to_string()); ofs_parameters.insert("ofs", "0".to_string()); @@ -80,7 +107,7 @@ impl SignalerChannelInner { [ ("VER", "8"), ("gsessionid", self.gsessionid.as_ref().unwrap()), - ("key", LIVE_CHAT_BASE_TANGO_KEY), + ("key", &self.tango_key), ("RID", &self.rid.to_string()), ("AID", &self.aid.to_string()), ("CVER", "22"), @@ -88,6 +115,9 @@ impl SignalerChannelInner { ("t", "1") ] )?) + // yes, this is required. why? who the fuck knows! but if you don't provide this, you get the typical google + // robot error complaining about an invalid request body when you GET GCM_SIGNALER_PSUB. yes, invalid request + // body, in a GET request. where the error actually refers to this POST request. because that makes sense. .header("X-WebChannel-Content-Type", "application/json+protobuf") .form(&ofs_parameters) .send() @@ -96,20 +126,21 @@ impl SignalerChannelInner { let mut ofs_res_line = ofs.bytes().await?.lines().nth(1).unwrap().unwrap(); let value: OwnedValue = unsafe { simd_json::from_str(&mut ofs_res_line) }?; let value = value.as_array().unwrap()[0].as_array().unwrap(); + // first value might be 1 if the request has an error, not entirely sure assert_eq!(value[0].as_usize().unwrap(), 0); let sid = value[1].as_array().unwrap()[1].as_str().unwrap(); self.sid = Some(sid.to_owned()); Ok(()) } - pub async fn get_session_stream(&self) -> Result { + pub async fn get_session_stream(&self) -> Result { Ok(super::get_http_client() .get(Url::parse_with_params( GCM_SIGNALER_PSUB, [ ("VER", "8"), ("gsessionid", self.gsessionid.as_ref().unwrap()), - ("key", LIVE_CHAT_BASE_TANGO_KEY), + ("key", &self.tango_key), ("RID", "rpc"), ("SID", self.sid.as_ref().unwrap()), ("AID", &self.aid.to_string()), @@ -124,78 +155,3 @@ impl SignalerChannelInner { .await?) } } - -#[derive(Debug)] -pub struct SignalerChannel { - inner: Arc> -} - -impl SignalerChannel { - pub async fn new(topic_id: impl ToString) -> Result { - Ok(SignalerChannel { - inner: Arc::new(Mutex::new(SignalerChannelInner::with_topic(topic_id))) - }) - } - - pub async fn new_from_cont(cont: &GetLiveChatResponse) -> Result { - Ok(SignalerChannel { - inner: Arc::new(Mutex::new(SignalerChannelInner::with_topic( - &cont.continuation_contents.as_ref().unwrap().live_chat_continuation.continuations[0] - .invalidation_continuation_data - .as_ref() - .unwrap() - .invalidation_id - .topic - ))) - }) - } - - pub async fn refresh_topic(&self, topic: impl ToString) { - self.inner.lock().await.topic = topic.to_string(); - } - - pub async fn spawn_event_subscriber(&self) -> Result<(broadcast::Receiver<()>, JoinHandle<()>), YouTubeError> { - let inner = Arc::clone(&self.inner); - { - let mut lock = inner.lock().await; - lock.choose_server().await?; - lock.init_session().await?; - } - let (sender, receiver) = broadcast::channel(128); - let handle = tokio::spawn(async move { - 'i: loop { - let mut req = { - let mut lock = inner.lock().await; - let _ = sender.send(()); - lock.reset(); - lock.choose_server().await.unwrap(); - lock.init_session().await.unwrap(); - lock.get_session_stream().await.unwrap() - }; - loop { - match req.chunk().await { - Ok(None) => break, - Ok(Some(s)) => { - let mut ofs_res_line = s.lines().nth(1).unwrap().unwrap(); - if let Ok(s) = unsafe { simd_json::from_str::(ofs_res_line.as_mut()) } { - let a = s.as_array().unwrap(); - { - inner.lock().await.aid = a[a.len() - 1].as_array().unwrap()[0].as_usize().unwrap(); - } - } - - if sender.send(()).is_err() { - break 'i; - } - } - Err(e) => { - eprintln!("{e:?}"); - break; - } - } - } - } - }); - Ok((receiver, handle)) - } -} diff --git a/src/youtube/types.rs b/src/youtube/types.rs deleted file mode 100644 index 9458493..0000000 --- a/src/youtube/types.rs +++ /dev/null @@ -1,334 +0,0 @@ -use chrono::{DateTime, Utc}; -use serde::{Deserialize, Serialize}; -use serde_aux::prelude::*; - -#[derive(Serialize, Debug)] -pub struct GetLiveChatBody { - context: GetLiveChatBodyContext, - continuation: String -} - -impl GetLiveChatBody { - pub fn new(continuation: impl Into, client_version: impl Into, client_name: impl Into) -> Self { - Self { - context: GetLiveChatBodyContext { - client: GetLiveChatBodyContextClient { - client_version: client_version.into(), - client_name: client_name.into() - } - }, - continuation: continuation.into() - } - } -} - -#[derive(Serialize, Debug)] -pub struct GetLiveChatBodyContext { - client: GetLiveChatBodyContextClient -} - -#[derive(Serialize, Debug)] -#[serde(rename_all = "camelCase")] -pub struct GetLiveChatBodyContextClient { - client_version: String, - client_name: String -} - -#[derive(Deserialize, Debug)] -#[serde(rename_all = "camelCase")] -pub struct GetLiveChatResponse { - pub response_context: Option, - pub tracking_params: Option, - pub continuation_contents: Option -} - -#[derive(Deserialize, Debug)] -#[serde(rename_all = "camelCase")] -pub struct GetLiveChatResponseContinuationContents { - pub live_chat_continuation: LiveChatContinuation -} -#[derive(Deserialize, Debug)] -pub struct LiveChatContinuation { - pub continuations: Vec, - pub actions: Option> -} - -#[derive(Deserialize, Debug)] -#[serde(rename_all = "camelCase")] -pub struct Continuation { - pub invalidation_continuation_data: Option, - pub timed_continuation_data: Option, - pub live_chat_replay_continuation_data: Option -} - -#[derive(Deserialize, Debug)] -#[serde(rename_all = "camelCase")] -pub struct LiveChatReplayContinuationData { - pub time_until_last_message_msec: usize, - pub continuation: String -} - -#[derive(Deserialize, Debug)] -#[serde(rename_all = "camelCase")] -pub struct InvalidationContinuationData { - pub invalidation_id: InvalidationId, - pub timeout_ms: usize, - pub continuation: String -} - -#[derive(Deserialize, Debug)] -#[serde(rename_all = "camelCase")] -pub struct InvalidationId { - pub object_source: usize, - pub object_id: String, - pub topic: String, - pub subscribe_to_gcm_topics: bool, - pub proto_creation_timestamp_ms: String -} - -#[derive(Deserialize, Debug)] -#[serde(rename_all = "camelCase")] -pub struct TimedContinuationData { - pub timeout_ms: usize, - pub continuation: String, - pub click_tracking_params: Option -} - -#[derive(Deserialize, Debug)] -#[serde(rename_all = "camelCase")] -pub struct Action { - pub add_chat_item_action: Option, - pub add_live_chat_ticker_item_action: Option, - pub replay_chat_item_action: Option -} - -#[derive(Deserialize, Debug)] -#[serde(rename_all = "camelCase")] -pub struct ReplayChatItemAction { - pub actions: Vec, - #[serde(deserialize_with = "deserialize_number_from_string")] - pub video_offset_time_msec: i64 -} - -// MessageRun -#[derive(Deserialize, Debug, Clone)] -#[serde(untagged)] -pub enum MessageRun { - MessageText { - text: String - }, - #[serde(rename_all = "camelCase")] - MessageEmoji { - emoji: Emoji, - variant_ids: Option> - } -} - -impl ToString for MessageRun { - fn to_string(&self) -> String { - match self { - Self::MessageText { text } => text.to_owned(), - Self::MessageEmoji { emoji, .. } => { - if let Some(true) = emoji.is_custom_emoji { - format!(":{}:", emoji.image.accessibility.accessibility_data.label) - } else { - emoji.image.accessibility.accessibility_data.label.to_owned() - } - } - } - } -} - -#[derive(Deserialize, Debug, Clone)] -#[serde(rename_all = "camelCase")] -pub struct Emoji { - pub emoji_id: String, - pub shortcuts: Option>, - pub search_terms: Option>, - pub supports_skin_tone: Option, - pub image: Image, - pub is_custom_emoji: Option -} - -#[derive(Deserialize, Debug, Clone)] -pub struct Image { - pub thumbnails: Vec, - pub accessibility: Accessibility -} - -#[derive(Deserialize, Debug, Clone)] -#[serde(rename_all = "camelCase")] -pub struct Accessibility { - pub accessibility_data: AccessibilityData -} - -#[derive(Deserialize, Debug, Clone)] -pub struct AccessibilityData { - pub label: String -} - -#[derive(Deserialize, Debug, Clone)] -pub struct Thumbnail { - pub url: String, - pub width: Option, - pub height: Option -} - -#[derive(Deserialize, Debug, Clone)] -#[serde(rename_all = "camelCase")] -pub struct AuthorBadge { - pub live_chat_author_badge_renderer: LiveChatAuthorBadgeRenderer -} - -#[derive(Deserialize, Debug, Clone)] -#[serde(rename_all = "camelCase")] -pub struct LiveChatAuthorBadgeRenderer { - pub custom_thumbnail: Option, - pub icon: Option, - pub tooltip: String, - pub accessibility: Accessibility -} - -#[derive(Deserialize, Debug, Clone)] -pub struct CustomThumbnail { - pub thumbnails: Vec -} - -#[derive(Deserialize, Debug, Clone)] -#[serde(rename_all = "camelCase")] -pub struct Icon { - pub icon_type: String -} - -#[derive(Deserialize, Debug, Clone)] -#[serde(rename_all = "camelCase")] -pub struct MessageRendererBase { - pub author_name: Option, - pub author_photo: AuthorPhoto, - pub author_badges: Option>, - pub context_menu_endpoint: ContextMenuEndpoint, - pub id: String, - #[serde(deserialize_with = "deserialize_datetime_utc_from_milliseconds")] - pub timestamp_usec: DateTime, - pub author_external_channel_id: String, - pub context_menu_accessibility: Accessibility -} - -#[derive(Deserialize, Debug, Clone)] -#[serde(rename_all = "camelCase")] -pub struct ContextMenuEndpoint { - pub click_tracking_params: Option, - pub command_metadata: CommandMetadata, - pub live_chat_item_context_menu_endpoint: LiveChatItemContextMenuEndpoint -} - -#[derive(Deserialize, Debug, Clone)] -pub struct LiveChatItemContextMenuEndpoint { - pub params: String -} - -#[derive(Deserialize, Debug, Clone)] -#[serde(rename_all = "camelCase")] -pub struct CommandMetadata { - pub web_command_metadata: WebCommandMetadata -} - -#[derive(Deserialize, Debug, Clone)] -#[serde(rename_all = "camelCase")] -pub struct WebCommandMetadata { - pub ignore_navigation: bool -} - -#[derive(Deserialize, Debug, Clone)] -pub struct AuthorPhoto { - pub thumbnails: Vec -} - -#[derive(Deserialize, Debug, Clone)] -#[serde(rename_all = "camelCase")] -pub struct AuthorName { - pub simple_text: String -} - -#[derive(Deserialize, Debug)] -pub struct LiveChatTextMessageRenderer { - #[serde(flatten)] - pub message_renderer_base: MessageRendererBase, - pub message: Option -} - -#[derive(Deserialize, Debug)] -pub struct Message { - pub runs: Vec -} - -#[derive(Deserialize, Debug)] -#[serde(rename_all = "camelCase")] -pub struct LiveChatPaidMessageRenderer { - #[serde(flatten)] - pub live_chat_text_message_renderer: LiveChatTextMessageRenderer, - pub purchase_amount_text: PurchaseAmountText, - pub header_background_color: isize, - pub header_text_color: isize, - pub body_background_color: isize, - pub body_text_color: isize, - pub author_name_text_color: isize -} - -#[derive(Deserialize, Debug)] -#[serde(rename_all = "camelCase")] -pub struct LiveChatPaidStickerRenderer { - #[serde(flatten)] - pub message_renderer_base: MessageRendererBase, - pub purchase_amount_text: PurchaseAmountText, - pub sticker: Sticker, - pub money_chip_background_color: isize, - pub money_chip_text_color: isize, - pub sticker_display_width: isize, - pub sticker_display_height: isize, - pub background_color: isize, - pub author_name_text_color: isize -} - -#[derive(Deserialize, Debug)] -pub struct Sticker { - pub thumbnails: Vec, - pub accessibility: Accessibility -} - -#[derive(Deserialize, Debug)] -#[serde(rename_all = "camelCase")] -pub struct PurchaseAmountText { - pub simple_text: String -} - -#[derive(Deserialize, Debug)] -#[serde(rename_all = "camelCase")] -pub struct LiveChatMembershipItemRenderer { - #[serde(flatten)] - pub message_renderer_base: MessageRendererBase, - pub header_sub_text: Option, - pub author_badges: Option> -} - -#[derive(Deserialize, Debug)] -pub struct HeaderSubText { - pub runs: Vec -} - -#[derive(Deserialize, Debug)] -#[serde(rename_all = "camelCase")] -pub struct AddChatItemAction { - pub item: ActionItem, - pub client_id: Option -} - -#[derive(Deserialize, Debug)] -#[serde(rename_all = "camelCase")] -pub struct ActionItem { - pub live_chat_text_message_renderer: Option, - pub live_chat_paid_message_renderer: Option, - pub live_chat_membership_item_renderer: Option, - pub live_chat_paid_sticker_renderer: Option, - pub live_chat_viewer_engagement_message_renderer: Option -} diff --git a/src/youtube/types/get_live_chat.rs b/src/youtube/types/get_live_chat.rs new file mode 100644 index 0000000..8753e89 --- /dev/null +++ b/src/youtube/types/get_live_chat.rs @@ -0,0 +1,260 @@ +// Copyright 2024 pyke.io +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use serde_aux::prelude::*; +use url::Url; + +use super::{Accessibility, CommandMetadata, Icon, ImageContainer, LocalizedText, UnlocalizedText}; +use crate::youtube::{ + get_http_client, + util::{SimdJsonRequestBody, SimdJsonResponseBody}, + ChatContext, Error, TANGO_LIVE_ENDPOINT, TANGO_REPLAY_ENDPOINT +}; + +#[derive(Serialize, Debug)] +pub struct GetLiveChatRequestBody { + context: GetLiveChatRequestBodyContext, + continuation: String +} + +impl GetLiveChatRequestBody { + pub(crate) fn new(continuation: impl Into, client_version: impl Into, client_name: impl Into) -> Self { + Self { + context: GetLiveChatRequestBodyContext { + client: GetLiveChatRequestBodyContextClient { + client_version: client_version.into(), + client_name: client_name.into() + } + }, + continuation: continuation.into() + } + } +} + +#[derive(Serialize, Debug)] +pub struct GetLiveChatRequestBodyContext { + client: GetLiveChatRequestBodyContextClient +} + +#[derive(Serialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct GetLiveChatRequestBodyContextClient { + client_version: String, + client_name: String +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct GetLiveChatResponse { + pub response_context: Option, + pub continuation_contents: GetLiveChatResponseContinuationContents +} + +impl GetLiveChatResponse { + pub async fn fetch(options: &ChatContext, continuation: impl AsRef) -> Result { + let body = GetLiveChatRequestBody::new(continuation.as_ref(), &options.client_version, "WEB"); + Ok(get_http_client() + .post(Url::parse_with_params( + if options.live_status.updates_live() { TANGO_LIVE_ENDPOINT } else { TANGO_REPLAY_ENDPOINT }, + [("key", options.api_key.as_str()), ("prettyPrint", "false")] + )?) + .simd_json(&body)? + .send() + .await? + .simd_json() + .await + .unwrap()) + } +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct GetLiveChatResponseContinuationContents { + pub live_chat_continuation: LiveChatContinuation +} + +#[derive(Deserialize, Debug)] +pub struct LiveChatContinuation { + pub continuations: Vec, + pub actions: Option> +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct ActionContainer { + #[serde(flatten)] + pub action: Action, + pub click_tracking_params: Option +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub enum Continuation { + #[serde(rename = "invalidationContinuationData")] + #[serde(rename_all = "camelCase")] + Invalidation { + invalidation_id: InvalidationId, + timeout_ms: usize, + continuation: String + }, + #[serde(rename = "timedContinuationData")] + #[serde(rename_all = "camelCase")] + Timed { timeout_ms: usize, continuation: String }, + #[serde(rename = "liveChatReplayContinuationData")] + #[serde(rename_all = "camelCase")] + Replay { time_until_last_message_msec: usize, continuation: String } +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct InvalidationId { + pub object_source: usize, + pub object_id: String, + pub topic: String, + pub subscribe_to_gcm_topics: bool, + pub proto_creation_timestamp_ms: String +} + +#[derive(Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub enum Action { + #[serde(rename = "addChatItemAction")] + #[serde(rename_all = "camelCase")] + AddChatItem { + item: ChatItem, + client_id: Option + }, + #[serde(rename = "removeChatItemAction")] + #[serde(rename_all = "camelCase")] + RemoveChatItem { + target_item_id: String + }, + #[serde(rename = "addLiveChatTickerItemAction")] + #[serde(rename_all = "camelCase")] + AddLiveChatTicker { + item: simd_json::OwnedValue + }, + #[serde(rename = "replayChatItemAction")] + #[serde(rename_all = "camelCase")] + ReplayChat { + actions: Vec, + #[serde(deserialize_with = "deserialize_number_from_string")] + video_offset_time_msec: i64 + }, + LiveChatReportModerationStateCommand(simd_json::OwnedValue) +} + +#[derive(Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct AuthorBadge { + pub live_chat_author_badge_renderer: LiveChatAuthorBadgeRenderer +} + +#[derive(Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct LiveChatAuthorBadgeRenderer { + pub custom_thumbnail: Option, + pub icon: Option, + pub tooltip: String, + pub accessibility: Accessibility +} + +#[derive(Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct MessageRendererBase { + pub author_name: Option, + pub author_photo: ImageContainer, + pub author_badges: Option>, + pub context_menu_endpoint: ContextMenuEndpoint, + pub id: String, + #[serde(deserialize_with = "deserialize_datetime_utc_from_milliseconds")] + pub timestamp_usec: DateTime, + pub author_external_channel_id: String, + pub context_menu_accessibility: Accessibility +} + +#[derive(Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct ContextMenuEndpoint { + pub command_metadata: CommandMetadata, + pub live_chat_item_context_menu_endpoint: LiveChatItemContextMenuEndpoint +} + +#[derive(Deserialize, Debug, Clone)] +pub struct LiveChatItemContextMenuEndpoint { + pub params: String +} + +#[derive(Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub enum ChatItem { + #[serde(rename = "liveChatTextMessageRenderer")] + #[serde(rename_all = "camelCase")] + TextMessage { + #[serde(flatten)] + message_renderer_base: MessageRendererBase, + message: Option + }, + #[serde(rename = "liveChatPaidMessageRenderer")] + #[serde(rename_all = "camelCase")] + Superchat { + #[serde(flatten)] + message_renderer_base: MessageRendererBase, + message: Option, + purchase_amount_text: UnlocalizedText, + header_background_color: isize, + header_text_color: isize, + body_background_color: isize, + body_text_color: isize, + author_name_text_color: isize + }, + #[serde(rename = "liveChatMembershipItemRenderer")] + #[serde(rename_all = "camelCase")] + MembershipItem { + #[serde(flatten)] + message_renderer_base: MessageRendererBase, + header_sub_text: Option, + author_badges: Option> + }, + #[serde(rename = "liveChatPaidStickerRenderer")] + #[serde(rename_all = "camelCase")] + PaidSticker { + #[serde(flatten)] + message_renderer_base: MessageRendererBase, + purchase_amount_text: UnlocalizedText, + sticker: ImageContainer, + money_chip_background_color: isize, + money_chip_text_color: isize, + sticker_display_width: isize, + sticker_display_height: isize, + background_color: isize, + author_name_text_color: isize + }, + #[serde(rename = "liveChatViewerEngagementMessageRenderer")] + ViewerEngagement { id: String } +} + +impl ChatItem { + pub fn id(&self) -> &str { + match self { + ChatItem::MembershipItem { message_renderer_base, .. } => &message_renderer_base.id, + ChatItem::PaidSticker { message_renderer_base, .. } => &message_renderer_base.id, + ChatItem::Superchat { message_renderer_base, .. } => &message_renderer_base.id, + ChatItem::TextMessage { message_renderer_base, .. } => &message_renderer_base.id, + ChatItem::ViewerEngagement { id } => id + } + } +} diff --git a/src/youtube/types/mod.rs b/src/youtube/types/mod.rs new file mode 100644 index 0000000..01898b8 --- /dev/null +++ b/src/youtube/types/mod.rs @@ -0,0 +1,106 @@ +// Copyright 2024 pyke.io +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use serde::Deserialize; +use simd_json::OwnedValue; + +pub mod get_live_chat; +pub mod streams_page; + +#[derive(Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct CommandMetadata { + pub web_command_metadata: OwnedValue +} + +#[derive(Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct UnlocalizedText { + pub simple_text: String, + pub accessibility: Option +} + +#[derive(Deserialize, Debug, Clone)] +#[serde(untagged)] +pub enum LocalizedRun { + Text { + text: String + }, + #[serde(rename_all = "camelCase")] + Emoji { + emoji: Emoji, + variant_ids: Option> + } +} + +impl LocalizedRun { + pub fn to_chat_string(&self) -> String { + match self { + Self::Text { text } => text.to_owned(), + Self::Emoji { emoji, .. } => { + if let Some(true) = emoji.is_custom_emoji { + format!(":{}:", emoji.image.accessibility.as_ref().unwrap().accessibility_data.label) + } else { + emoji.image.accessibility.as_ref().unwrap().accessibility_data.label.to_owned() + } + } + } + } +} + +#[derive(Deserialize, Debug, Clone)] +pub struct LocalizedText { + pub runs: Vec +} + +#[derive(Deserialize, Debug, Clone)] +pub struct ImageContainer { + pub thumbnails: Vec, + pub accessibility: Option +} + +#[derive(Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct Accessibility { + pub accessibility_data: AccessibilityData +} + +#[derive(Deserialize, Debug, Clone)] +pub struct AccessibilityData { + pub label: String +} + +#[derive(Deserialize, Debug, Clone)] +pub struct Thumbnail { + pub url: String, + pub width: Option, + pub height: Option +} + +#[derive(Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct Emoji { + pub emoji_id: String, + pub shortcuts: Option>, + pub search_terms: Option>, + pub supports_skin_tone: Option, + pub image: ImageContainer, + pub is_custom_emoji: Option +} + +#[derive(Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct Icon { + pub icon_type: String +} diff --git a/src/youtube/types/streams_page.rs b/src/youtube/types/streams_page.rs new file mode 100644 index 0000000..b0bb631 --- /dev/null +++ b/src/youtube/types/streams_page.rs @@ -0,0 +1,131 @@ +// Copyright 2024 pyke.io +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use serde::Deserialize; + +use super::{Accessibility, CommandMetadata, ImageContainer, LocalizedText}; + +#[derive(Debug, Deserialize)] +pub struct YouTubeInitialData { + pub contents: PageContentsRenderer +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum PageContentsRenderer { + TwoColumnBrowseResultsRenderer { tabs: Vec } +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum TabItemRenderer { + TabRenderer { + endpoint: FeedEndpoint, + title: String, + #[serde(default)] + selected: bool, + content: Option + }, + ExpandableTabRenderer {} +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct FeedEndpoint { + pub browse_endpoint: BrowseEndpoint, + pub command_metadata: CommandMetadata +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct BrowseEndpoint { + pub browse_id: String, + pub params: String, + pub canonical_base_url: String +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum FeedContentsRenderer { + RichGridRenderer { contents: Vec, header: FeedHeaderRenderer }, + SectionListRenderer { contents: Vec } +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum RichGridItem { + #[serde(rename_all = "camelCase")] + RichItemRenderer { content: RichItemContent }, + #[serde(rename_all = "camelCase")] + ContinuationItemRenderer { trigger: ContinuationItemTrigger } +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum RichItemContent { + #[serde(rename_all = "camelCase")] + VideoRenderer { + description_snippet: LocalizedText, + thumbnail: ImageContainer, + thumbnail_overlays: Vec, + video_id: String + } +} + +#[derive(Debug, Deserialize)] +pub enum ThumbnailOverlay { + #[serde(rename = "thumbnailOverlayTimeStatusRenderer")] + TimeStatus { + style: VideoTimeStatus // text: UnlocalizedText + }, + #[serde(rename = "thumbnailOverlayToggleButtonRenderer")] + #[serde(rename_all = "camelCase")] + ToggleButton { + is_toggled: Option, + toggled_accessibility: Accessibility, + toggled_tooltip: String, + untoggled_accessibility: Accessibility, + untoggled_tooltip: String + }, + #[serde(rename = "thumbnailOverlayNowPlayingRenderer")] + NowPlaying { text: LocalizedText } +} + +#[derive(Debug, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub enum VideoTimeStatus { + Upcoming, + Live, + Default +} + +#[derive(Debug, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub enum ContinuationItemTrigger { + ContinuationTriggerOnItemShown +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum FeedHeaderRenderer { + #[serde(rename_all = "camelCase")] + FeedFilterChipBarRenderer { contents: Vec, style_type: String } +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum FeedFilterChip { + #[serde(rename_all = "camelCase")] + ChipCloudChipRenderer { is_selected: bool } +} diff --git a/src/youtube/util.rs b/src/youtube/util.rs index 98f57e2..1293102 100644 --- a/src/youtube/util.rs +++ b/src/youtube/util.rs @@ -1,29 +1,43 @@ +// Copyright 2024 pyke.io +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + use std::future::Future; use reqwest::{RequestBuilder, Response}; use serde::{de::DeserializeOwned, Serialize}; -use super::YouTubeError; +use super::Error; pub trait SimdJsonResponseBody { - fn simd_json(self) -> impl Future>; + fn simd_json(self) -> impl Future>; } impl SimdJsonResponseBody for Response { - async fn simd_json(self) -> Result { + async fn simd_json(self) -> Result { let mut full = self.bytes().await?.to_vec(); Ok(simd_json::from_slice(&mut full)?) } } pub trait SimdJsonRequestBody { - fn simd_json(self, json: &T) -> Result + fn simd_json(self, json: &T) -> Result where Self: Sized; } impl SimdJsonRequestBody for RequestBuilder { - fn simd_json(self, json: &T) -> Result { + fn simd_json(self, json: &T) -> Result { Ok(self.body(simd_json::to_vec(json)?)) } } From 38f93c011920495bcb6e9497f4684d575e47ca1f Mon Sep 17 00:00:00 2001 From: "Carson M." Date: Thu, 8 Feb 2024 22:34:28 -0600 Subject: [PATCH 8/9] fix vod --- src/youtube/mod.rs | 59 +++++++++++------------------- src/youtube/types/get_live_chat.rs | 25 +++++++++++-- 2 files changed, 44 insertions(+), 40 deletions(-) diff --git a/src/youtube/mod.rs b/src/youtube/mod.rs index a8a488b..a95d5e4 100644 --- a/src/youtube/mod.rs +++ b/src/youtube/mod.rs @@ -69,7 +69,8 @@ impl<'r> ActionChunk<'r> { let continuation_token = match &response.continuation_contents.live_chat_continuation.continuations[0] { Continuation::Invalidation { continuation, .. } => continuation.to_owned(), Continuation::Timed { continuation, .. } => continuation.to_owned(), - Continuation::Replay { continuation, .. } => continuation.to_owned() + Continuation::Replay { continuation, .. } => continuation.to_owned(), + Continuation::PlayerSeek { .. } => return Err(Error::EndOfContinuation) }; let signaler_topic = match &response.continuation_contents.live_chat_continuation.continuations[0] { Continuation::Invalidation { invalidation_id, .. } => Some(invalidation_id.topic.to_owned()), @@ -93,7 +94,7 @@ impl<'r> ActionChunk<'r> { .ok_or(Error::EndOfContinuation)? .into_iter() .flat_map(|f| match f.action { - Action::ReplayChat { actions, .. } => actions, + Action::ReplayChat { actions, .. } => actions.into_iter().map(|f| f.action).collect(), f => vec![f] }) .collect() @@ -159,8 +160,8 @@ pub async fn stream(options: &ChatContext) -> Result { for action in actions { - if let Action::AddChatItem { .. } = action { - yield_tx.send(Ok(action.to_owned())).await; + if let Action::AddChatItem { .. } = action.action { + yield_tx.send(Ok(action.action.to_owned())).await; } } } @@ -187,8 +188,8 @@ pub async fn stream(options: &ChatContext) -> Result { for action in actions { - if let Action::AddChatItem { .. } = action { - yield_tx.send(Ok(action.to_owned())).await; + if let Action::AddChatItem { .. } = action.action { + yield_tx.send(Ok(action.action.to_owned())).await; } } } @@ -232,8 +233,8 @@ pub async fn stream(options: &ChatContext) -> Result { for action in actions { - if let Action::AddChatItem { .. } = action { - yield_tx.send(Ok(action.to_owned())).await; + if let Action::AddChatItem { .. } = action.action { + yield_tx.send(Ok(action.action.to_owned())).await; } } } @@ -255,38 +256,17 @@ pub async fn stream(options: &ChatContext) -> Result { - let chunk = ActionChunk::new(initial_chat, options).unwrap(); - for action in chunk.iter() { - match action { - Action::AddChatItem { .. } => { - yield_tx.send(Ok(action.to_owned())).await; - } - Action::ReplayChat { actions, .. } => { - for action in actions { - if let Action::AddChatItem { .. } = action { - yield_tx.send(Ok(action.to_owned())).await; - } - } - } - action => { - yield_tx.send(Ok(action.to_owned())).await; - } - } - } - - while let Some(Ok(chunk)) = chunk.cont().await { + let mut chunk = ActionChunk::new(initial_chat, options).unwrap(); + loop { for action in chunk.iter() { match action { - Action::AddChatItem { item, .. } => { - if !seen_messages.contains(item.id()) { - yield_tx.send(Ok(action.to_owned())).await; - seen_messages.insert(item.id().to_owned()); - } + Action::AddChatItem { .. } => { + yield_tx.send(Ok(action.to_owned())).await; } Action::ReplayChat { actions, .. } => { for action in actions { - if let Action::AddChatItem { .. } = action { - yield_tx.send(Ok(action.to_owned())).await; + if let Action::AddChatItem { .. } = action.action { + yield_tx.send(Ok(action.action.to_owned())).await; } } } @@ -295,6 +275,10 @@ pub async fn stream(options: &ChatContext) -> Result chunk = e, + _ => break + } } } Continuation::Timed { timeout_ms, .. } => { @@ -311,8 +295,8 @@ pub async fn stream(options: &ChatContext) -> Result { for action in actions { - if let Action::AddChatItem { .. } = action { - yield_tx.send(Ok(action.to_owned())).await; + if let Action::AddChatItem { .. } = action.action { + yield_tx.send(Ok(action.action.to_owned())).await; } } } @@ -328,6 +312,7 @@ pub async fn stream(options: &ChatContext) -> Result panic!("player seek should not be first continuation") } }))) } diff --git a/src/youtube/types/get_live_chat.rs b/src/youtube/types/get_live_chat.rs index 8753e89..03d9cfc 100644 --- a/src/youtube/types/get_live_chat.rs +++ b/src/youtube/types/get_live_chat.rs @@ -92,7 +92,7 @@ pub struct LiveChatContinuation { pub actions: Option> } -#[derive(Deserialize, Debug)] +#[derive(Deserialize, Debug, Clone)] #[serde(rename_all = "camelCase")] pub struct ActionContainer { #[serde(flatten)] @@ -115,7 +115,10 @@ pub enum Continuation { Timed { timeout_ms: usize, continuation: String }, #[serde(rename = "liveChatReplayContinuationData")] #[serde(rename_all = "camelCase")] - Replay { time_until_last_message_msec: usize, continuation: String } + Replay { time_until_last_message_msec: usize, continuation: String }, + #[serde(rename = "playerSeekContinuationData")] + #[serde(rename_all = "camelCase")] + PlayerSeek { continuation: String } } #[derive(Deserialize, Debug)] @@ -150,7 +153,7 @@ pub enum Action { #[serde(rename = "replayChatItemAction")] #[serde(rename_all = "camelCase")] ReplayChat { - actions: Vec, + actions: Vec, #[serde(deserialize_with = "deserialize_number_from_string")] video_offset_time_msec: i64 }, @@ -243,6 +246,20 @@ pub enum ChatItem { background_color: isize, author_name_text_color: isize }, + #[serde(rename = "liveChatSponsorshipsGiftPurchaseAnnouncementRenderer")] + #[serde(rename_all = "camelCase")] + MembershipGift { + id: String, + #[serde(flatten)] + data: simd_json::OwnedValue + }, + #[serde(rename = "liveChatSponsorshipsGiftRedemptionAnnouncementRenderer")] + #[serde(rename_all = "camelCase")] + MembershipGiftRedemption { + id: String, + #[serde(flatten)] + data: simd_json::OwnedValue + }, #[serde(rename = "liveChatViewerEngagementMessageRenderer")] ViewerEngagement { id: String } } @@ -254,6 +271,8 @@ impl ChatItem { ChatItem::PaidSticker { message_renderer_base, .. } => &message_renderer_base.id, ChatItem::Superchat { message_renderer_base, .. } => &message_renderer_base.id, ChatItem::TextMessage { message_renderer_base, .. } => &message_renderer_base.id, + ChatItem::MembershipGift { id, .. } => id, + ChatItem::MembershipGiftRedemption { id, .. } => id, ChatItem::ViewerEngagement { id } => id } } From d1fa005777cee3e5218a0cbc70f52b7e84a3202d Mon Sep 17 00:00:00 2001 From: "Carson M." Date: Thu, 8 Feb 2024 22:43:38 -0600 Subject: [PATCH 9/9] fix timestamps --- src/youtube/types/get_live_chat.rs | 4 ++-- src/youtube/types/mod.rs | 17 ++++++++++++++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/youtube/types/get_live_chat.rs b/src/youtube/types/get_live_chat.rs index 03d9cfc..566593d 100644 --- a/src/youtube/types/get_live_chat.rs +++ b/src/youtube/types/get_live_chat.rs @@ -17,7 +17,7 @@ use serde::{Deserialize, Serialize}; use serde_aux::prelude::*; use url::Url; -use super::{Accessibility, CommandMetadata, Icon, ImageContainer, LocalizedText, UnlocalizedText}; +use super::{deserialize_datetime_utc_from_microseconds, Accessibility, CommandMetadata, Icon, ImageContainer, LocalizedText, UnlocalizedText}; use crate::youtube::{ get_http_client, util::{SimdJsonRequestBody, SimdJsonResponseBody}, @@ -183,7 +183,7 @@ pub struct MessageRendererBase { pub author_badges: Option>, pub context_menu_endpoint: ContextMenuEndpoint, pub id: String, - #[serde(deserialize_with = "deserialize_datetime_utc_from_milliseconds")] + #[serde(deserialize_with = "deserialize_datetime_utc_from_microseconds")] pub timestamp_usec: DateTime, pub author_external_channel_id: String, pub context_menu_accessibility: Accessibility diff --git a/src/youtube/types/mod.rs b/src/youtube/types/mod.rs index 01898b8..96845e2 100644 --- a/src/youtube/types/mod.rs +++ b/src/youtube/types/mod.rs @@ -12,7 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use serde::Deserialize; +use serde::{de::Error, Deserialize, Deserializer}; +use serde_aux::field_attributes::deserialize_number_from_string; use simd_json::OwnedValue; pub mod get_live_chat; @@ -104,3 +105,17 @@ pub struct Emoji { pub struct Icon { pub icon_type: String } + +pub fn deserialize_datetime_utc_from_microseconds<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de> +{ + use chrono::prelude::*; + + let number = deserialize_number_from_string::(deserializer)?; + let seconds = number / 1_000_000; + let micros = (number % 1_000_000) as u32; + let nanos = micros * 1_000; + + Ok(Utc.from_utc_datetime(&NaiveDateTime::from_timestamp_opt(seconds, nanos).ok_or_else(|| D::Error::custom("Couldn't parse the timestamp"))?)) +}