Revert "cache available channel log dates"

This reverts commit c2d34ff048.

.dockerignore

@@ -3,4 +3,3 @@
 config.json
 **node_modules/
 Dockerfile
-ch-data/

Cargo.lock

@@ -119,12 +119,6 @@ version = "1.0.71"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8"
 
-[[package]]
-name = "arc-swap"
-version = "1.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6"
-
 [[package]]
 name = "async-trait"
 version = "0.1.68"
@@ -1441,7 +1435,6 @@ version = "0.1.0"
 dependencies = [
  "aide",
  "anyhow",
- "arc-swap",
  "axum",
  "chrono",
  "clap",

Cargo.toml

@@ -51,7 +51,6 @@ twitch_api2 = { version = "0.6.1", features = [
     "twitch_oauth2",
 ] }
 twitch = { git = "https://github.com/jprochazk/twitch-rs", features = ["simd"] }
-arc-swap = "1.6.0"
 
 [dev-dependencies]
 pretty_assertions = "1.3.0"

docs/CONFIG.md

@@ -8,7 +8,6 @@ Available options:
 - `clickhouseUsername` (string): Clickhouse username.
 - `clickhousePassword` (string): Clickhouse password.
 - `clickhouseFlushInterval` (number): Interval (in seconds) of how often messages should be flushed to the database. A lower value means that logs are available sooner at the expensive of higher database load. Defaults to 10.
-- `channelLogsDateCacheInterval` (number): Interval (in seconds) of how often the channel log dates cache should be updated. Defaults to 120.
 - `listenAddress` (string): Listening address for the web server. Defaults to `0.0.0.0:8025`.
 - `channels` (array of strings): List of channel ids to be logged.
 - `clientId` (string): Twitch client id.

src/app/mod.rs

@@ -1,22 +1,11 @@
 pub mod cache;
 
 use self::cache::UsersCache;
-use crate::{
+use crate::{config::Config, error::Error, Result};
-    config::Config,
-    db::{read_all_available_channel_logs, AvailableChannelLogs},
-    error::Error,
-    Result,
-};
 use anyhow::Context;
-use arc_swap::ArcSwap;
 use dashmap::DashSet;
-use std::{
+use std::{collections::HashMap, sync::Arc};
-    collections::HashMap,
+use tracing::debug;
-    sync::Arc,
-    time::{Duration, Instant},
-};
-use tokio::time::sleep;
-use tracing::{debug, error};
 use twitch_api2::{helix::users::GetUsersRequest, twitch_oauth2::AppAccessToken, HelixClient};
 
 #[derive(Clone)]
@@ -25,7 +14,6 @@ pub struct App {
     pub token: Arc<AppAccessToken>,
     pub users: UsersCache,
     pub optout_codes: Arc<DashSet<String>>,
-    pub channel_log_dates_cache: Arc<ArcSwap<AvailableChannelLogs>>,
     pub db: Arc<clickhouse::Client>,
     pub config: Arc<Config>,
 }
@@ -119,31 +107,4 @@ impl App {
 
         Ok(())
     }
-
-    pub fn start_channel_log_dates_cacher(&self) {
-        let app = self.clone();
-
-        tokio::spawn(async move {
-            loop {
-                sleep(Duration::from_secs(
-                    app.config.channel_logs_date_cache_interval,
-                ))
-                .await;
-
-                let started_at = Instant::now();
-                match read_all_available_channel_logs(&app.db).await {
-                    Ok(new_dates) => {
-                        app.channel_log_dates_cache.store(Arc::new(new_dates));
-                        debug!(
-                            "Updated channel log dates cache (took {}ms)",
-                            started_at.elapsed().as_millis()
-                        );
-                    }
-                    Err(err) => {
-                        error!("Could not update available channel logs: {err}");
-                    }
-                }
-            }
-        });
-    }
 }

src/config.rs

@@ -16,8 +16,6 @@ pub struct Config {
     pub clickhouse_password: Option<String>,
     #[serde(default = "clickhouse_flush_interval")]
     pub clickhouse_flush_interval: u64,
-    #[serde(default = "channel_log_dates_cache_interval")]
-    pub channel_logs_date_cache_interval: u64,
     #[serde(default = "default_listen_address")]
     pub listen_address: String,
     pub channels: RwLock<HashSet<String>>,
@@ -52,7 +50,3 @@ fn default_listen_address() -> String {
 fn clickhouse_flush_interval() -> u64 {
     10
 }
-
-fn channel_log_dates_cache_interval() -> u64 {
-    120
-}

src/db/mod.rs

@@ -1,8 +1,6 @@
 pub mod schema;
 pub mod writer;
 
-use std::collections::HashMap;
-
 use crate::{
     error::Error,
     logs::{
@@ -17,8 +15,6 @@ use clickhouse::Client;
 use rand::{seq::IteratorRandom, thread_rng};
 use tracing::info;
 
-pub type AvailableChannelLogs = HashMap<String, Vec<AvailableLogDate>>;
-
 pub async fn read_channel(
     db: &Client,
     channel_id: &str,
@@ -56,27 +52,32 @@ pub async fn read_user(
     LogsStream::new_cursor(cursor).await
 }
 
-pub async fn read_all_available_channel_logs(db: &Client) -> Result<AvailableChannelLogs> {
+pub async fn read_available_channel_logs(
-    let all_dates = db
+    db: &Client,
-        .query("SELECT channel_id, toDateTime(toStartOfDay(timestamp)) AS date FROM message GROUP BY date, channel_id ORDER BY date DESC")
+    channel_id: &str,
-        .fetch_all::<(String, i32)>().await?;
+) -> Result<Vec<AvailableLogDate>> {
+    let timestamps: Vec<i32> = db
+        .query(
+            "SELECT toDateTime(toStartOfDay(timestamp)) AS date FROM message WHERE channel_id = ? GROUP BY date ORDER BY date DESC",
+        )
+        .bind(channel_id)
+        .fetch_all().await?;
 
-    let mut channels: AvailableChannelLogs = HashMap::new();
+    let dates = timestamps
+        .into_iter()
+        .map(|timestamp| {
+            let naive =
+                NaiveDateTime::from_timestamp_opt(timestamp.into(), 0).expect("Invalid DateTime");
 
-    for (channel_id, timestamp) in all_dates {
+            AvailableLogDate {
-        let naive =
+                year: naive.year().to_string(),
-            NaiveDateTime::from_timestamp_opt(timestamp.into(), 0).expect("Invalid DateTime");
+                month: naive.month().to_string(),
+                day: Some(naive.day().to_string()),
+            }
+        })
+        .collect();
 
-        let available_log = AvailableLogDate {
+    Ok(dates)
-            year: naive.year().to_string(),
-            month: naive.month().to_string(),
-            day: Some(naive.day().to_string()),
-        };
-
-        channels.entry(channel_id).or_default().push(available_log);
-    }
-
-    Ok(channels)
 }
 
 pub async fn read_available_user_logs(

src/main.rs

@@ -13,7 +13,6 @@ pub type ShutdownRx = watch::Receiver<()>;
 
 use anyhow::{anyhow, Context};
 use app::App;
-use arc_swap::ArcSwap;
 use args::{Args, Command};
 use clap::Parser;
 use config::Config;
@@ -38,7 +37,7 @@ use twitch_api2::{
 };
 use twitch_irc::login::StaticLoginCredentials;
 
-use crate::{app::cache::UsersCache, db::read_all_available_channel_logs};
+use crate::app::cache::UsersCache;
 
 const SHUTDOWN_TIMEOUT_SECONDS: u64 = 8;
 
@@ -93,11 +92,6 @@ async fn run(config: Config, db: clickhouse::Client) -> anyhow::Result<()> {
     )
     .await?;
 
-    let channel_log_dates_cache = read_all_available_channel_logs(&db)
-        .await
-        .context("Could not fetch available log dates")?;
-    let channel_log_dates_cache = Arc::new(ArcSwap::new(Arc::new(channel_log_dates_cache)));
-
     let app = App {
         helix_client,
         token: Arc::new(token),
@@ -105,11 +99,8 @@ async fn run(config: Config, db: clickhouse::Client) -> anyhow::Result<()> {
         config: Arc::new(config),
         db: Arc::new(db),
         optout_codes: Arc::default(),
-        channel_log_dates_cache,
     };
 
-    app.start_channel_log_dates_cacher();
-
     let login_credentials = StaticLoginCredentials::anonymous();
     let mut bot_handle = tokio::spawn(bot::run(
         login_credentials,

src/web/handlers.rs

@@ -8,8 +8,8 @@ use super::{
 use crate::{
     app::App,
     db::{
-        read_available_user_logs, read_channel, read_random_channel_line, read_random_user_line,
+        read_available_channel_logs, read_available_user_logs, read_channel,
-        read_user,
+        read_random_channel_line, read_random_user_line, read_user,
     },
     error::Error,
     logs::{
@@ -159,8 +159,7 @@ pub async fn list_available_logs(
         read_available_user_logs(&app.db, &channel_id, &user_id).await?
     } else {
         app.check_opted_out(&channel_id, None)?;
-        let cache = app.channel_log_dates_cache.load();
+        read_available_channel_logs(&app.db, &channel_id).await?
-        cache.get(&channel_id).cloned().ok_or(Error::NotFound)?
     };
 
     if !available_logs.is_empty() {

src/web/schema.rs

@@ -135,7 +135,7 @@ pub struct AvailableLogs {
     pub available_logs: Vec<AvailableLogDate>,
 }
 
-#[derive(Serialize, JsonSchema, Clone)]
+#[derive(Serialize, JsonSchema)]
 pub struct AvailableLogDate {
     pub year: String,
     pub month: String,