Fijxu
/
invidious
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

limit feeds and delete materialized views
Invidious CI / build - crystal: ${{ matrix.crystal }}, stable: ${{ matrix.stable }} (1.10.1, true) (push) Waiting to run Details
Invidious CI / build - crystal: ${{ matrix.crystal }}, stable: ${{ matrix.stable }} (1.7.3, true) (push) Waiting to run Details
Invidious CI / build - crystal: ${{ matrix.crystal }}, stable: ${{ matrix.stable }} (1.8.2, true) (push) Waiting to run Details
Invidious CI / build - crystal: ${{ matrix.crystal }}, stable: ${{ matrix.stable }} (1.9.2, true) (push) Waiting to run Details
Invidious CI / build - crystal: ${{ matrix.crystal }}, stable: ${{ matrix.stable }} (nightly, false) (push) Waiting to run Details
Invidious CI / build-docker (push) Waiting to run Details
Invidious CI / build-docker-arm64 (push) Waiting to run Details
Build and release container / release (push) Has been cancelled Details

This commit is contained in:
Emilien Devos 2023-07-26 15:43:43 +02:00 committed by zzls Selfhost
parent 8e530beaa1
commit de3a444939
10 changed files with 41 additions and 125 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
config/migrate-scripts/migrate-db-8bc91ce.sh Normal file
View File

@ -0,0 +1,6 @@
CREATE INDEX channel_videos_ucid_published_idx
ON public.channel_videos
USING btree
(ucid COLLATE pg_catalog."default", published);
DROP INDEX channel_videos_ucid_idx;

8
config/sql/channel_videos.sql
View File

@ -19,12 +19,12 @@ CREATE TABLE IF NOT EXISTS public.channel_videos
GRANT ALL ON TABLE public.channel_videos TO current_user;
-- Index: public.channel_videos_ucid_idx
-- Index: public.channel_videos_ucid_published_idx
-- DROP INDEX public.channel_videos_ucid_idx;
-- DROP INDEX public.channel_videos_ucid_published_idx;
CREATE INDEX IF NOT EXISTS channel_videos_ucid_idx
CREATE INDEX IF NOT EXISTS channel_videos_ucid_published_idx
ON public.channel_videos
USING btree
(ucid COLLATE pg_catalog."default");
(ucid COLLATE pg_catalog."default", published);

1
kubernetes/values.yaml
View File

@ -49,7 +49,6 @@ postgresql:
# Adapted from ../config/config.yml
config:
channel_threads: 1
feed_threads: 1
db:
user: kemal
password: kemal

12
src/invidious.cr
View File

@ -109,14 +109,6 @@ Kemal.config.extra_options do |parser|
exit
end
end
parser.on("-f THREADS", "--feed-threads=THREADS", "Number of threads for refreshing feeds (default: #{CONFIG.feed_threads})") do |number|
begin
CONFIG.feed_threads = number.to_i
rescue ex
puts "THREADS must be integer"
exit
end
end
parser.on("-o OUTPUT", "--output=OUTPUT", "Redirect output (default: #{CONFIG.output})") do |output|
CONFIG.output = output
end
@ -165,10 +157,6 @@ if CONFIG.channel_threads > 0
Invidious::Jobs.register Invidious::Jobs::RefreshChannelsJob.new(PG_DB)
end
if CONFIG.feed_threads > 0
Invidious::Jobs.register Invidious::Jobs::RefreshFeedsJob.new(PG_DB)
end
DECRYPT_FUNCTION = DecryptFunction.new(CONFIG.decrypt_polling)
if CONFIG.decrypt_polling
Invidious::Jobs.register Invidious::Jobs::UpdateDecryptFunctionJob.new

2
src/invidious/config.cr
View File

@ -62,8 +62,6 @@ class Config
# Time interval between two executions of the job that crawls channel videos (subscriptions update).
@[YAML::Field(converter: Preferences::TimeSpanConverter)]
property channel_refresh_interval : Time::Span = 30.minutes
# Number of threads to use for updating feeds
property feed_threads : Int32 = 1
# Log file path or STDOUT
property output : String = "STDOUT"
# Default log level, valid YAML values are ints and strings, see src/invidious/helpers/logger.cr

75
src/invidious/jobs/refresh_feeds_job.cr
View File

@ -1,75 +0,0 @@
class Invidious::Jobs::RefreshFeedsJob < Invidious::Jobs::BaseJob
private getter db : DB::Database
def initialize(@db)
end
def begin
max_fibers = CONFIG.feed_threads
active_fibers = 0
active_channel = ::Channel(Bool).new
loop do
db.query("SELECT email FROM users WHERE feed_needs_update = true OR feed_needs_update IS NULL") do |rs|
rs.each do
email = rs.read(String)
view_name = "subscriptions_#{sha256(email)}"
if active_fibers >= max_fibers
if active_channel.receive
active_fibers -= 1
end
end
active_fibers += 1
spawn do
begin
# Drop outdated views
column_array = Invidious::Database.get_column_array(db, view_name)
ChannelVideo.type_array.each_with_index do |name, i|
if name != column_array[i]?
LOGGER.info("RefreshFeedsJob: DROP MATERIALIZED VIEW #{view_name}")
db.exec("DROP MATERIALIZED VIEW #{view_name}")
raise "view does not exist"
end
end
if !db.query_one("SELECT pg_get_viewdef('#{view_name}')", as: String).includes? "WHERE ((cv.ucid = ANY (u.subscriptions))"
LOGGER.info("RefreshFeedsJob: Materialized view #{view_name} is out-of-date, recreating...")
db.exec("DROP MATERIALIZED VIEW #{view_name}")
end
db.exec("REFRESH MATERIALIZED VIEW #{view_name}")
db.exec("UPDATE users SET feed_needs_update = false WHERE email = $1", email)
rescue ex
# Rename old views
begin
legacy_view_name = "subscriptions_#{sha256(email)[0..7]}"
db.exec("SELECT * FROM #{legacy_view_name} LIMIT 0")
LOGGER.info("RefreshFeedsJob: RENAME MATERIALIZED VIEW #{legacy_view_name}")
db.exec("ALTER MATERIALIZED VIEW #{legacy_view_name} RENAME TO #{view_name}")
rescue ex
begin
# While iterating through, we may have an email stored from a deleted account
if db.query_one?("SELECT true FROM users WHERE email = $1", email, as: Bool)
LOGGER.info("RefreshFeedsJob: CREATE #{view_name}")
db.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(email)}")
db.exec("UPDATE users SET feed_needs_update = false WHERE email = $1", email)
end
rescue ex
LOGGER.error("RefreshFeedJobs: REFRESH #{email} : #{ex.message}")
end
end
end
active_channel.send(true)
end
end
end
sleep 5.seconds
Fiber.yield
end
end
end

2
src/invidious/routes/account.cr
View File

@ -123,10 +123,8 @@ module Invidious::Routes::Account
return error_template(400, ex)
end
view_name = "subscriptions_#{sha256(user.email)}"
Invidious::Database::Users.delete(user)
Invidious::Database::SessionIDs.delete(email: user.email)
PG_DB.exec("DROP MATERIALIZED VIEW #{view_name}")
env.request.cookies.each do |cookie|
cookie.expires = Time.utc(1990, 1, 1)

3
src/invidious/routes/login.cr
View File

@ -160,9 +160,6 @@ module Invidious::Routes::Login
Invidious::Database::Users.insert(user)
Invidious::Database::SessionIDs.insert(sid, email)
view_name = "subscriptions_#{sha256(user.email)}"
PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}")
env.response.cookies["SID"] = Invidious::User::Cookies.sid(CONFIG.domain, sid)
if env.request.cookies["PREFS"]?

18
src/invidious/search/processors.cr
View File

@ -37,18 +37,18 @@ module Invidious::Search
# Search inside of user subscriptions
def subscriptions(query : Query, user : Invidious::User) : Array(ChannelVideo)
view_name = "subscriptions_#{sha256(user.email)}"
return PG_DB.query_all("
SELECT id,title,published,updated,ucid,author,length_seconds
FROM (
SELECT *,
to_tsvector(#{view_name}.title) ||
to_tsvector(#{view_name}.author)
as document
FROM #{view_name}
) v_search WHERE v_search.document @@ plainto_tsquery($1) LIMIT 20 OFFSET $2;",
query.text, (query.page - 1) * 20,
SELECT cv.*,
to_tsvector(cv.title) ||
to_tsvector(cv.author) AS document
FROM channel_videos cv
JOIN users ON cv.ucid = any(users.subscriptions)
WHERE users.email = $1 AND published > now() - interval '1 month'
ORDER BY published
) v_search WHERE v_search.document @@ plainto_tsquery($2) LIMIT 20 OFFSET $3;",
user.email, query.text, (query.page - 1) * 20,
as: ChannelVideo
)
end

39
src/invidious/users.cr
View File

@ -27,7 +27,6 @@ def get_subscription_feed(user, max_results = 40, page = 1)
offset = (page - 1) * limit
notifications = Invidious::Database::Users.select_notifications(user)
view_name = "subscriptions_#{sha256(user.email)}"
if user.preferences.notifications_only && !notifications.empty?
# Only show notifications
@ -53,33 +52,39 @@ def get_subscription_feed(user, max_results = 40, page = 1)
# Show latest video from a channel that a user hasn't watched
# "unseen_only" isn't really correct here, more accurate would be "unwatched_only"
if user.watched.empty?
values = "'{}'"
else
values = "VALUES #{user.watched.map { |id| %(('#{id}')) }.join(",")}"
end
videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} WHERE NOT id = ANY (#{values}) ORDER BY ucid, published DESC", as: ChannelVideo)
# "SELECT cv.* FROM channel_videos cv JOIN users ON cv.ucid = any(users.subscriptions) WHERE users.email = $1 AND published > now() - interval '1 month' ORDER BY published DESC"
# "SELECT DISTINCT ON (cv.ucid) cv.* FROM channel_videos cv JOIN users ON cv.ucid = any(users.subscriptions) WHERE users.email = ? AND NOT cv.id = any(users.watched) AND published > now() - interval '1 month' ORDER BY ucid, published DESC"
videos = PG_DB.query_all("SELECT DISTINCT ON (cv.ucid) cv.* " \
"FROM channel_videos cv " \
"JOIN users ON cv.ucid = any(users.subscriptions) " \
"WHERE users.email = $1 AND NOT cv.id = any(users.watched) AND published > now() - interval '1 month' " \
"ORDER BY ucid, published DESC", user.email, as: ChannelVideo)
else
# Show latest video from each channel
videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} ORDER BY ucid, published DESC", as: ChannelVideo)
videos = PG_DB.query_all("SELECT DISTINCT ON (cv.ucid) cv.* " \
"FROM channel_videos cv " \
"JOIN users ON cv.ucid = any(users.subscriptions) " \
"WHERE users.email = $1 AND published > now() - interval '1 month' " \
"ORDER BY ucid, published DESC", user.email, as: ChannelVideo)
end
videos.sort_by!(&.published).reverse!
else
if user.preferences.unseen_only
# Only show unwatched
if user.watched.empty?
values = "'{}'"
else
values = "VALUES #{user.watched.map { |id| %(('#{id}')) }.join(",")}"
end
videos = PG_DB.query_all("SELECT * FROM #{view_name} WHERE NOT id = ANY (#{values}) ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: ChannelVideo)
videos = PG_DB.query_all("SELECT cv.* " \
"FROM channel_videos cv " \
"JOIN users ON cv.ucid = any(users.subscriptions) " \
"WHERE users.email = $1 AND NOT cv.id = any(users.watched) AND published > now() - interval '1 month' " \
"ORDER BY published DESC LIMIT $2 OFFSET $3", user.email, limit, offset, as: ChannelVideo)
else
# Sort subscriptions as normal
videos = PG_DB.query_all("SELECT * FROM #{view_name} ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: ChannelVideo)
videos = PG_DB.query_all("SELECT cv.* " \
"FROM channel_videos cv " \
"JOIN users ON cv.ucid = any(users.subscriptions) " \
"WHERE users.email = $1 AND published > now() - interval '1 month' " \
"ORDER BY published DESC LIMIT $2 OFFSET $3", user.email, limit, offset, as: ChannelVideo)
end
end