forked from iv-org/invidious
-
Notifications
You must be signed in to change notification settings - Fork 0
/
invidious.cr
238 lines (193 loc) · 7.79 KB
/
invidious.cr
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
# "Invidious" (which is an alternative front-end to YouTube)
# Copyright (C) 2019 Omar Roth
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
require "digest/md5"
require "file_utils"
# Require kemal, kilt, then our own overrides
require "kemal"
require "kilt"
require "./ext/kemal_content_for.cr"
require "./ext/kemal_static_file_handler.cr"
require "athena-negotiation"
require "openssl/hmac"
require "option_parser"
require "sqlite3"
require "xml"
require "yaml"
require "compress/zip"
require "protodec/utils"
require "./invidious/database/*"
require "./invidious/database/migrations/*"
require "./invidious/http_server/*"
require "./invidious/helpers/*"
require "./invidious/yt_backend/*"
require "./invidious/frontend/*"
require "./invidious/videos/*"
require "./invidious/jsonify/**"
require "./invidious/*"
require "./invidious/comments/*"
require "./invidious/channels/*"
require "./invidious/user/*"
require "./invidious/search/*"
require "./invidious/routes/**"
require "./invidious/jobs/**"
# Declare the base namespace for invidious
module Invidious
end
# Simple alias to make code easier to read
alias IV = Invidious
CONFIG = Config.load
HMAC_KEY = CONFIG.hmac_key
PG_DB = DB.open CONFIG.database_url
ARCHIVE_URL = URI.parse("https://archive.org")
PUBSUB_URL = URI.parse("https://pubsubhubbub.appspot.com")
REDDIT_URL = URI.parse("https://www.reddit.com")
YT_URL = URI.parse("https://www.youtube.com")
HOST_URL = make_host_url(Kemal.config)
CHARS_SAFE = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"
TEST_IDS = {"AgbeGFYluEA", "BaW_jenozKc", "a9LDPn-MO4I", "ddFvjfvPnqk", "iqKdEhx-dD4"}
MAX_ITEMS_PER_PAGE = 1500
REQUEST_HEADERS_WHITELIST = {"accept", "accept-encoding", "cache-control", "content-length", "if-none-match", "range"}
RESPONSE_HEADERS_BLACKLIST = {"access-control-allow-origin", "alt-svc", "server"}
HTTP_CHUNK_SIZE = 10485760 # ~10MB
CURRENT_BRANCH = {{ "#{`git branch | sed -n '/* /s///p'`.strip}" }}
CURRENT_COMMIT = {{ "#{`git rev-list HEAD --max-count=1 --abbrev-commit`.strip}" }}
CURRENT_VERSION = {{ "#{`git log -1 --format=%ci | awk '{print $1}' | sed s/-/./g`.strip}" }}
# This is used to determine the `?v=` on the end of file URLs (for cache busting). We
# only need to expire modified assets, so we can use this to find the last commit that changes
# any assets
ASSET_COMMIT = {{ "#{`git rev-list HEAD --max-count=1 --abbrev-commit -- assets`.strip}" }}
SOFTWARE = {
"name" => "invidious",
"version" => "#{CURRENT_VERSION}-#{CURRENT_COMMIT}",
"branch" => "#{CURRENT_BRANCH}",
}
YT_POOL = YoutubeConnectionPool.new(YT_URL, capacity: CONFIG.pool_size)
# CLI
Kemal.config.extra_options do |parser|
parser.banner = "Usage: invidious [arguments]"
parser.on("-c THREADS", "--channel-threads=THREADS", "Number of threads for refreshing channels (default: #{CONFIG.channel_threads})") do |number|
begin
CONFIG.channel_threads = number.to_i
rescue ex
puts "THREADS must be integer"
exit
end
end
parser.on("-f THREADS", "--feed-threads=THREADS", "Number of threads for refreshing feeds (default: #{CONFIG.feed_threads})") do |number|
begin
CONFIG.feed_threads = number.to_i
rescue ex
puts "THREADS must be integer"
exit
end
end
parser.on("-o OUTPUT", "--output=OUTPUT", "Redirect output (default: #{CONFIG.output})") do |output|
CONFIG.output = output
end
parser.on("-l LEVEL", "--log-level=LEVEL", "Log level, one of #{LogLevel.values} (default: #{CONFIG.log_level})") do |log_level|
CONFIG.log_level = LogLevel.parse(log_level)
end
parser.on("-v", "--version", "Print version") do
puts SOFTWARE.to_pretty_json
exit
end
parser.on("--migrate", "Run any migrations (beta, use at your own risk!!") do
Invidious::Database::Migrator.new(PG_DB).migrate
exit
end
end
Kemal::CLI.new ARGV
if CONFIG.output.upcase != "STDOUT"
FileUtils.mkdir_p(File.dirname(CONFIG.output))
end
OUTPUT = CONFIG.output.upcase == "STDOUT" ? STDOUT : File.open(CONFIG.output, mode: "a")
LOGGER = Invidious::LogHandler.new(OUTPUT, CONFIG.log_level)
# Check table integrity
Invidious::Database.check_integrity(CONFIG)
{% if !flag?(:skip_videojs_download) %}
# Resolve player dependencies. This is done at compile time.
#
# Running the script by itself would show some colorful feedback while this doesn't.
# Perhaps we should just move the script to runtime in order to get that feedback?
{% puts "\nChecking player dependencies, this may take more than 20 minutes... If it is stuck, check your internet connection.\n" %}
{% if flag?(:minified_player_dependencies) %}
{% puts run("../scripts/fetch-player-dependencies.cr", "--minified").stringify %}
{% else %}
{% puts run("../scripts/fetch-player-dependencies.cr").stringify %}
{% end %}
{% puts "\nDone checking player dependencies, now compiling Invidious...\n" %}
{% end %}
# Start jobs
if CONFIG.channel_threads > 0
Invidious::Jobs.register Invidious::Jobs::RefreshChannelsJob.new(PG_DB)
end
if CONFIG.feed_threads > 0
Invidious::Jobs.register Invidious::Jobs::RefreshFeedsJob.new(PG_DB)
end
DECRYPT_FUNCTION = DecryptFunction.new(CONFIG.decrypt_polling)
if CONFIG.decrypt_polling
Invidious::Jobs.register Invidious::Jobs::UpdateDecryptFunctionJob.new
end
if CONFIG.statistics_enabled
Invidious::Jobs.register Invidious::Jobs::StatisticsRefreshJob.new(PG_DB, SOFTWARE)
end
if (CONFIG.use_pubsub_feeds.is_a?(Bool) && CONFIG.use_pubsub_feeds.as(Bool)) || (CONFIG.use_pubsub_feeds.is_a?(Int32) && CONFIG.use_pubsub_feeds.as(Int32) > 0)
Invidious::Jobs.register Invidious::Jobs::SubscribeToFeedsJob.new(PG_DB, HMAC_KEY)
end
if CONFIG.popular_enabled
Invidious::Jobs.register Invidious::Jobs::PullPopularVideosJob.new(PG_DB)
end
CONNECTION_CHANNEL = ::Channel({Bool, ::Channel(PQ::Notification)}).new(32)
Invidious::Jobs.register Invidious::Jobs::NotificationJob.new(CONNECTION_CHANNEL, CONFIG.database_url)
Invidious::Jobs.register Invidious::Jobs::ClearExpiredItemsJob.new
Invidious::Jobs.start_all
def popular_videos
Invidious::Jobs::PullPopularVideosJob::POPULAR_VIDEOS.get
end
# Routing
before_all do |env|
Invidious::Routes::BeforeAll.handle(env)
end
Invidious::Routing.register_all
error 404 do |env|
Invidious::Routes::ErrorRoutes.error_404(env)
end
error 500 do |env, ex|
error_template(500, ex)
end
static_headers do |response|
response.headers.add("Cache-Control", "max-age=2629800")
end
# Init Kemal
public_folder "assets"
Kemal.config.powered_by_header = false
add_handler FilteredCompressHandler.new
add_handler APIHandler.new
add_handler AuthHandler.new
add_handler DenyFrame.new
add_context_storage_type(Array(String))
add_context_storage_type(Preferences)
add_context_storage_type(Invidious::User)
Kemal.config.logger = LOGGER
Kemal.config.host_binding = Kemal.config.host_binding != "0.0.0.0" ? Kemal.config.host_binding : CONFIG.host_binding
Kemal.config.port = Kemal.config.port != 3000 ? Kemal.config.port : CONFIG.port
Kemal.config.app_name = "Invidious"
# Use in kemal's production mode.
# Users can also set the KEMAL_ENV environmental variable for this to be set automatically.
{% if flag?(:release) || flag?(:production) %}
Kemal.config.env = "production" if !ENV.has_key?("KEMAL_ENV")
{% end %}
Kemal.run