diff --git a/.dockerignore b/.dockerignore index 09d1db9..0c2f0b4 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,4 +3,3 @@ config.json **node_modules/ Dockerfile -ch-data/ diff --git a/Cargo.lock b/Cargo.lock index 3739ee5..f7eaa5e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -119,12 +119,6 @@ version = "1.0.71" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" -[[package]] -name = "arc-swap" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" - [[package]] name = "async-trait" version = "0.1.68" @@ -1441,7 +1435,6 @@ version = "0.1.0" dependencies = [ "aide", "anyhow", - "arc-swap", "axum", "chrono", "clap", diff --git a/Cargo.toml b/Cargo.toml index 5976f2f..f8bc529 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,7 +51,6 @@ twitch_api2 = { version = "0.6.1", features = [ "twitch_oauth2", ] } twitch = { git = "https://github.com/jprochazk/twitch-rs", features = ["simd"] } -arc-swap = "1.6.0" [dev-dependencies] pretty_assertions = "1.3.0" diff --git a/docs/CONFIG.md b/docs/CONFIG.md index e7daae4..50ac435 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -8,7 +8,6 @@ Available options: - `clickhouseUsername` (string): Clickhouse username. - `clickhousePassword` (string): Clickhouse password. - `clickhouseFlushInterval` (number): Interval (in seconds) of how often messages should be flushed to the database. A lower value means that logs are available sooner at the expensive of higher database load. Defaults to 10. -- `channelLogsDateCacheInterval` (number): Interval (in seconds) of how often the channel log dates cache should be updated. Defaults to 120. - `listenAddress` (string): Listening address for the web server. Defaults to `0.0.0.0:8025`. - `channels` (array of strings): List of channel ids to be logged. - `clientId` (string): Twitch client id. diff --git a/src/app/mod.rs b/src/app/mod.rs index 7375071..13b2dee 100644 --- a/src/app/mod.rs +++ b/src/app/mod.rs @@ -1,22 +1,11 @@ pub mod cache; use self::cache::UsersCache; -use crate::{ - config::Config, - db::{read_all_available_channel_logs, AvailableChannelLogs}, - error::Error, - Result, -}; +use crate::{config::Config, error::Error, Result}; use anyhow::Context; -use arc_swap::ArcSwap; use dashmap::DashSet; -use std::{ - collections::HashMap, - sync::Arc, - time::{Duration, Instant}, -}; -use tokio::time::sleep; -use tracing::{debug, error}; +use std::{collections::HashMap, sync::Arc}; +use tracing::debug; use twitch_api2::{helix::users::GetUsersRequest, twitch_oauth2::AppAccessToken, HelixClient}; #[derive(Clone)] @@ -25,7 +14,6 @@ pub struct App { pub token: Arc, pub users: UsersCache, pub optout_codes: Arc>, - pub channel_log_dates_cache: Arc>, pub db: Arc, pub config: Arc, } @@ -119,31 +107,4 @@ impl App { Ok(()) } - - pub fn start_channel_log_dates_cacher(&self) { - let app = self.clone(); - - tokio::spawn(async move { - loop { - sleep(Duration::from_secs( - app.config.channel_logs_date_cache_interval, - )) - .await; - - let started_at = Instant::now(); - match read_all_available_channel_logs(&app.db).await { - Ok(new_dates) => { - app.channel_log_dates_cache.store(Arc::new(new_dates)); - debug!( - "Updated channel log dates cache (took {}ms)", - started_at.elapsed().as_millis() - ); - } - Err(err) => { - error!("Could not update available channel logs: {err}"); - } - } - } - }); - } } diff --git a/src/config.rs b/src/config.rs index 3dfb264..c972dad 100644 --- a/src/config.rs +++ b/src/config.rs @@ -16,8 +16,6 @@ pub struct Config { pub clickhouse_password: Option, #[serde(default = "clickhouse_flush_interval")] pub clickhouse_flush_interval: u64, - #[serde(default = "channel_log_dates_cache_interval")] - pub channel_logs_date_cache_interval: u64, #[serde(default = "default_listen_address")] pub listen_address: String, pub channels: RwLock>, @@ -52,7 +50,3 @@ fn default_listen_address() -> String { fn clickhouse_flush_interval() -> u64 { 10 } - -fn channel_log_dates_cache_interval() -> u64 { - 120 -} diff --git a/src/db/mod.rs b/src/db/mod.rs index 6892e86..bbebf0b 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -1,8 +1,6 @@ pub mod schema; pub mod writer; -use std::collections::HashMap; - use crate::{ error::Error, logs::{ @@ -17,8 +15,6 @@ use clickhouse::Client; use rand::{seq::IteratorRandom, thread_rng}; use tracing::info; -pub type AvailableChannelLogs = HashMap>; - pub async fn read_channel( db: &Client, channel_id: &str, @@ -56,27 +52,32 @@ pub async fn read_user( LogsStream::new_cursor(cursor).await } -pub async fn read_all_available_channel_logs(db: &Client) -> Result { - let all_dates = db - .query("SELECT channel_id, toDateTime(toStartOfDay(timestamp)) AS date FROM message GROUP BY date, channel_id ORDER BY date DESC") - .fetch_all::<(String, i32)>().await?; +pub async fn read_available_channel_logs( + db: &Client, + channel_id: &str, +) -> Result> { + let timestamps: Vec = db + .query( + "SELECT toDateTime(toStartOfDay(timestamp)) AS date FROM message WHERE channel_id = ? GROUP BY date ORDER BY date DESC", + ) + .bind(channel_id) + .fetch_all().await?; - let mut channels: AvailableChannelLogs = HashMap::new(); + let dates = timestamps + .into_iter() + .map(|timestamp| { + let naive = + NaiveDateTime::from_timestamp_opt(timestamp.into(), 0).expect("Invalid DateTime"); - for (channel_id, timestamp) in all_dates { - let naive = - NaiveDateTime::from_timestamp_opt(timestamp.into(), 0).expect("Invalid DateTime"); + AvailableLogDate { + year: naive.year().to_string(), + month: naive.month().to_string(), + day: Some(naive.day().to_string()), + } + }) + .collect(); - let available_log = AvailableLogDate { - year: naive.year().to_string(), - month: naive.month().to_string(), - day: Some(naive.day().to_string()), - }; - - channels.entry(channel_id).or_default().push(available_log); - } - - Ok(channels) + Ok(dates) } pub async fn read_available_user_logs( diff --git a/src/main.rs b/src/main.rs index ebcad39..818f34c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -13,7 +13,6 @@ pub type ShutdownRx = watch::Receiver<()>; use anyhow::{anyhow, Context}; use app::App; -use arc_swap::ArcSwap; use args::{Args, Command}; use clap::Parser; use config::Config; @@ -38,7 +37,7 @@ use twitch_api2::{ }; use twitch_irc::login::StaticLoginCredentials; -use crate::{app::cache::UsersCache, db::read_all_available_channel_logs}; +use crate::app::cache::UsersCache; const SHUTDOWN_TIMEOUT_SECONDS: u64 = 8; @@ -93,11 +92,6 @@ async fn run(config: Config, db: clickhouse::Client) -> anyhow::Result<()> { ) .await?; - let channel_log_dates_cache = read_all_available_channel_logs(&db) - .await - .context("Could not fetch available log dates")?; - let channel_log_dates_cache = Arc::new(ArcSwap::new(Arc::new(channel_log_dates_cache))); - let app = App { helix_client, token: Arc::new(token), @@ -105,11 +99,8 @@ async fn run(config: Config, db: clickhouse::Client) -> anyhow::Result<()> { config: Arc::new(config), db: Arc::new(db), optout_codes: Arc::default(), - channel_log_dates_cache, }; - app.start_channel_log_dates_cacher(); - let login_credentials = StaticLoginCredentials::anonymous(); let mut bot_handle = tokio::spawn(bot::run( login_credentials, diff --git a/src/web/handlers.rs b/src/web/handlers.rs index 8e74100..ae726b1 100644 --- a/src/web/handlers.rs +++ b/src/web/handlers.rs @@ -8,8 +8,8 @@ use super::{ use crate::{ app::App, db::{ - read_available_user_logs, read_channel, read_random_channel_line, read_random_user_line, - read_user, + read_available_channel_logs, read_available_user_logs, read_channel, + read_random_channel_line, read_random_user_line, read_user, }, error::Error, logs::{ @@ -159,8 +159,7 @@ pub async fn list_available_logs( read_available_user_logs(&app.db, &channel_id, &user_id).await? } else { app.check_opted_out(&channel_id, None)?; - let cache = app.channel_log_dates_cache.load(); - cache.get(&channel_id).cloned().ok_or(Error::NotFound)? + read_available_channel_logs(&app.db, &channel_id).await? }; if !available_logs.is_empty() { diff --git a/src/web/schema.rs b/src/web/schema.rs index 5a53fa0..974d556 100644 --- a/src/web/schema.rs +++ b/src/web/schema.rs @@ -135,7 +135,7 @@ pub struct AvailableLogs { pub available_logs: Vec, } -#[derive(Serialize, JsonSchema, Clone)] +#[derive(Serialize, JsonSchema)] pub struct AvailableLogDate { pub year: String, pub month: String,