From 45f19a4aa77f5e9d75246ac38e40836987062315 Mon Sep 17 00:00:00 2001 From: Miroslav Hettes Date: Sat, 7 Oct 2023 18:23:56 +0200 Subject: [PATCH] Remove que from project --- Gemfile | 5 +- Gemfile.lock | 13 - config.ru | 8 - config/application.rb | 2 - config/initializers/queweb.rb | 5 - config/routes.rb | 4 - db/migrate/20190423214925_add_que.rb | 4 +- db/migrate/20190915104202_upgrade_que.rb | 2 +- db/migrate/20220322180237_upgrade_que_v5.rb | 2 +- db/migrate/20230325092744_bump_que_version.rb | 2 +- db/migrate/20231007153003_remove_que_jobs.rb | 11 + db/structure.sql | 332 +----------------- 12 files changed, 19 insertions(+), 371 deletions(-) delete mode 100644 config/initializers/queweb.rb create mode 100644 db/migrate/20231007153003_remove_que_jobs.rb diff --git a/Gemfile b/Gemfile index 88a7007b..6ca5029e 100644 --- a/Gemfile +++ b/Gemfile @@ -34,10 +34,7 @@ gem 'jbuilder' # Use Capistrano for deployment # gem 'capistrano-rails', group: :development -# Gems for tracking the statuses of jobs -gem 'que' -gem 'que-web' - +# Workers gem 'good_job' # Reduces boot times through caching; required in config/boot.rb diff --git a/Gemfile.lock b/Gemfile.lock index b1ebbe51..cfd2202e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -252,8 +252,6 @@ GEM minitest (5.20.0) msgpack (1.7.2) multi_xml (0.6.0) - mustermann (3.0.0) - ruby2_keywords (~> 0.0.1) net-imap (0.4.0) date net-protocol @@ -310,10 +308,6 @@ GEM public_suffix (5.0.3) puma (6.4.0) nio4r (~> 2.0) - que (2.2.1) - que-web (0.10.0) - que (>= 1) - sinatra raabro (1.4.0) racc (1.7.1) rack (2.2.8) @@ -417,11 +411,6 @@ GEM simplecov_json_formatter (~> 0.1) simplecov-html (0.12.3) simplecov_json_formatter (0.1.4) - sinatra (3.1.0) - mustermann (~> 3.0) - rack (~> 2.2, >= 2.2.4) - rack-protection (= 3.1.0) - tilt (~> 2.0) sitemap_generator (6.3.0) builder (~> 3.0) snaky_hash (2.0.1) @@ -511,8 +500,6 @@ DEPENDENCIES premailer-rails pry-rails puma - que - que-web rails (~> 7.0.8) rails-controller-testing (~> 1.0) rails-i18n diff --git a/config.ru b/config.ru index 65042a6f..899bdacb 100644 --- a/config.ru +++ b/config.ru @@ -1,11 +1,3 @@ # This file is used by Rack-based servers to start the application. - -# Added que-web gem for showing job tracking -require 'que/web' - -map '/admin/que' do - run Que::Web -end - require_relative 'config/environment' run Rails.application diff --git a/config/application.rb b/config/application.rb index ec8bdf22..9c1f66a9 100644 --- a/config/application.rb +++ b/config/application.rb @@ -63,5 +63,3 @@ class Application < Rails::Application } end end - -require "que/active_record/model" diff --git a/config/initializers/queweb.rb b/config/initializers/queweb.rb deleted file mode 100644 index 1cbdb610..00000000 --- a/config/initializers/queweb.rb +++ /dev/null @@ -1,5 +0,0 @@ -# Authentication allowing only admin to view the que status -Que::Web.use(Rack::Auth::Basic) do |user, password| - [user, password] == [Rails.application.config_for(:app).dig(:admin, :username), - Rails.application.config_for(:app).dig(:admin, :password)] -end diff --git a/config/routes.rb b/config/routes.rb index c5c97b97..b324703e 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -42,10 +42,6 @@ put :hide, on: :member end - # Route for que - require 'que/web' - mount Que::Web => '/admin/que' - mount GoodJob::Engine => 'good_job' end diff --git a/db/migrate/20190423214925_add_que.rb b/db/migrate/20190423214925_add_que.rb index 55cfd215..f6836f0a 100644 --- a/db/migrate/20190423214925_add_que.rb +++ b/db/migrate/20190423214925_add_que.rb @@ -3,11 +3,11 @@ class AddQue < ActiveRecord::Migration[4.2] def self.up # The current version as of this migration's creation. - Que.migrate! :version => 3 + # Que.migrate! :version => 3 end def self.down # Completely removes Que's job queue. - Que.migrate! :version => 0 + # Que.migrate! :version => 0 end end diff --git a/db/migrate/20190915104202_upgrade_que.rb b/db/migrate/20190915104202_upgrade_que.rb index 079ec0fa..fc167fe2 100644 --- a/db/migrate/20190915104202_upgrade_que.rb +++ b/db/migrate/20190915104202_upgrade_que.rb @@ -1,5 +1,5 @@ class UpgradeQue < ActiveRecord::Migration[6.0] def up - Que.migrate!(version: 4) + # Que.migrate!(version: 4) end end diff --git a/db/migrate/20220322180237_upgrade_que_v5.rb b/db/migrate/20220322180237_upgrade_que_v5.rb index 47871006..2aaa10a3 100644 --- a/db/migrate/20220322180237_upgrade_que_v5.rb +++ b/db/migrate/20220322180237_upgrade_que_v5.rb @@ -1,6 +1,6 @@ class UpgradeQueV5 < ActiveRecord::Migration[6.1] def up - Que.migrate!(version: 5) + # Que.migrate!(version: 5) end end diff --git a/db/migrate/20230325092744_bump_que_version.rb b/db/migrate/20230325092744_bump_que_version.rb index a60bd024..4ebcbfde 100644 --- a/db/migrate/20230325092744_bump_que_version.rb +++ b/db/migrate/20230325092744_bump_que_version.rb @@ -1,5 +1,5 @@ class BumpQueVersion < ActiveRecord::Migration[6.1] def up - Que.migrate!(version: 7) + # Que.migrate!(version: 7) end end diff --git a/db/migrate/20231007153003_remove_que_jobs.rb b/db/migrate/20231007153003_remove_que_jobs.rb new file mode 100644 index 00000000..089b36dd --- /dev/null +++ b/db/migrate/20231007153003_remove_que_jobs.rb @@ -0,0 +1,11 @@ +class RemoveQueJobs < ActiveRecord::Migration[7.0] + def up + drop_table :que_jobs, force: :cascade, if_exists: true + drop_table :que_values, force: :cascade, if_exists: true + drop_table :que_lockers, force: :cascade, if_exists: true + + execute 'DROP FUNCTION IF EXISTS que_validate_tags CASCADE' + execute 'DROP FUNCTION IF EXISTS que_state_notify CASCADE' + execute 'DROP FUNCTION IF EXISTS que_job_notify CASCADE' + end +end diff --git a/db/structure.sql b/db/structure.sql index dad6648a..5ac160b5 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -23,217 +23,10 @@ CREATE SCHEMA code_list; CREATE SCHEMA upvs; --- --- Name: que_validate_tags(jsonb); Type: FUNCTION; Schema: public; Owner: - --- - -CREATE FUNCTION public.que_validate_tags(tags_array jsonb) RETURNS boolean - LANGUAGE sql - AS $$ - SELECT bool_and( - jsonb_typeof(value) = 'string' - AND - char_length(value::text) <= 100 - ) - FROM jsonb_array_elements(tags_array) -$$; - - SET default_tablespace = ''; SET default_table_access_method = heap; --- --- Name: que_jobs; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.que_jobs ( - priority smallint DEFAULT 100 NOT NULL, - run_at timestamp with time zone DEFAULT now() NOT NULL, - id bigint NOT NULL, - job_class text NOT NULL, - error_count integer DEFAULT 0 NOT NULL, - last_error_message text, - queue text DEFAULT 'default'::text NOT NULL, - last_error_backtrace text, - finished_at timestamp with time zone, - expired_at timestamp with time zone, - args jsonb DEFAULT '[]'::jsonb NOT NULL, - data jsonb DEFAULT '{}'::jsonb NOT NULL, - job_schema_version integer NOT NULL, - kwargs jsonb DEFAULT '{}'::jsonb NOT NULL, - CONSTRAINT error_length CHECK (((char_length(last_error_message) <= 500) AND (char_length(last_error_backtrace) <= 10000))), - CONSTRAINT job_class_length CHECK ((char_length( -CASE job_class - WHEN 'ActiveJob::QueueAdapters::QueAdapter::JobWrapper'::text THEN ((args -> 0) ->> 'job_class'::text) - ELSE job_class -END) <= 200)), - CONSTRAINT queue_length CHECK ((char_length(queue) <= 100)), - CONSTRAINT valid_args CHECK ((jsonb_typeof(args) = 'array'::text)), - CONSTRAINT valid_data CHECK (((jsonb_typeof(data) = 'object'::text) AND ((NOT (data ? 'tags'::text)) OR ((jsonb_typeof((data -> 'tags'::text)) = 'array'::text) AND (jsonb_array_length((data -> 'tags'::text)) <= 5) AND public.que_validate_tags((data -> 'tags'::text)))))) -) -WITH (fillfactor='90'); - - --- --- Name: TABLE que_jobs; Type: COMMENT; Schema: public; Owner: - --- - -COMMENT ON TABLE public.que_jobs IS '7'; - - --- --- Name: que_determine_job_state(public.que_jobs); Type: FUNCTION; Schema: public; Owner: - --- - -CREATE FUNCTION public.que_determine_job_state(job public.que_jobs) RETURNS text - LANGUAGE sql - AS $$ - SELECT - CASE - WHEN job.expired_at IS NOT NULL THEN 'expired' - WHEN job.finished_at IS NOT NULL THEN 'finished' - WHEN job.error_count > 0 THEN 'errored' - WHEN job.run_at > CURRENT_TIMESTAMP THEN 'scheduled' - ELSE 'ready' - END -$$; - - --- --- Name: que_job_notify(); Type: FUNCTION; Schema: public; Owner: - --- - -CREATE FUNCTION public.que_job_notify() RETURNS trigger - LANGUAGE plpgsql - AS $$ - DECLARE - locker_pid integer; - sort_key json; - BEGIN - -- Don't do anything if the job is scheduled for a future time. - IF NEW.run_at IS NOT NULL AND NEW.run_at > now() THEN - RETURN null; - END IF; - - -- Pick a locker to notify of the job's insertion, weighted by their number - -- of workers. Should bounce pseudorandomly between lockers on each - -- invocation, hence the md5-ordering, but still touch each one equally, - -- hence the modulo using the job_id. - SELECT pid - INTO locker_pid - FROM ( - SELECT *, last_value(row_number) OVER () + 1 AS count - FROM ( - SELECT *, row_number() OVER () - 1 AS row_number - FROM ( - SELECT * - FROM public.que_lockers ql, generate_series(1, ql.worker_count) AS id - WHERE - listening AND - queues @> ARRAY[NEW.queue] AND - ql.job_schema_version = NEW.job_schema_version - ORDER BY md5(pid::text || id::text) - ) t1 - ) t2 - ) t3 - WHERE NEW.id % count = row_number; - - IF locker_pid IS NOT NULL THEN - -- There's a size limit to what can be broadcast via LISTEN/NOTIFY, so - -- rather than throw errors when someone enqueues a big job, just - -- broadcast the most pertinent information, and let the locker query for - -- the record after it's taken the lock. The worker will have to hit the - -- DB in order to make sure the job is still visible anyway. - SELECT row_to_json(t) - INTO sort_key - FROM ( - SELECT - 'job_available' AS message_type, - NEW.queue AS queue, - NEW.priority AS priority, - NEW.id AS id, - -- Make sure we output timestamps as UTC ISO 8601 - to_char(NEW.run_at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS run_at - ) t; - - PERFORM pg_notify('que_listener_' || locker_pid::text, sort_key::text); - END IF; - - RETURN null; - END -$$; - - --- --- Name: que_state_notify(); Type: FUNCTION; Schema: public; Owner: - --- - -CREATE FUNCTION public.que_state_notify() RETURNS trigger - LANGUAGE plpgsql - AS $$ - DECLARE - row record; - message json; - previous_state text; - current_state text; - BEGIN - IF TG_OP = 'INSERT' THEN - previous_state := 'nonexistent'; - current_state := public.que_determine_job_state(NEW); - row := NEW; - ELSIF TG_OP = 'DELETE' THEN - previous_state := public.que_determine_job_state(OLD); - current_state := 'nonexistent'; - row := OLD; - ELSIF TG_OP = 'UPDATE' THEN - previous_state := public.que_determine_job_state(OLD); - current_state := public.que_determine_job_state(NEW); - - -- If the state didn't change, short-circuit. - IF previous_state = current_state THEN - RETURN null; - END IF; - - row := NEW; - ELSE - RAISE EXCEPTION 'Unrecognized TG_OP: %', TG_OP; - END IF; - - SELECT row_to_json(t) - INTO message - FROM ( - SELECT - 'job_change' AS message_type, - row.id AS id, - row.queue AS queue, - - coalesce(row.data->'tags', '[]'::jsonb) AS tags, - - to_char(row.run_at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS run_at, - to_char(now() AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS time, - - CASE row.job_class - WHEN 'ActiveJob::QueueAdapters::QueAdapter::JobWrapper' THEN - coalesce( - row.args->0->>'job_class', - 'ActiveJob::QueueAdapters::QueAdapter::JobWrapper' - ) - ELSE - row.job_class - END AS job_class, - - previous_state AS previous_state, - current_state AS current_state - ) t; - - PERFORM pg_notify('que_state', message::text); - - RETURN null; - END -$$; - - -- -- Name: countries; Type: TABLE; Schema: code_list; Owner: - -- @@ -1006,55 +799,6 @@ CREATE SEQUENCE public.pg_search_documents_id_seq ALTER SEQUENCE public.pg_search_documents_id_seq OWNED BY public.pg_search_documents.id; --- --- Name: que_jobs_id_seq; Type: SEQUENCE; Schema: public; Owner: - --- - -CREATE SEQUENCE public.que_jobs_id_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - - --- --- Name: que_jobs_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - --- - -ALTER SEQUENCE public.que_jobs_id_seq OWNED BY public.que_jobs.id; - - --- --- Name: que_lockers; Type: TABLE; Schema: public; Owner: - --- - -CREATE UNLOGGED TABLE public.que_lockers ( - pid integer NOT NULL, - worker_count integer NOT NULL, - worker_priorities integer[] NOT NULL, - ruby_pid integer NOT NULL, - ruby_hostname text NOT NULL, - queues text[] NOT NULL, - listening boolean NOT NULL, - job_schema_version integer DEFAULT 1, - CONSTRAINT valid_queues CHECK (((array_ndims(queues) = 1) AND (array_length(queues, 1) IS NOT NULL))), - CONSTRAINT valid_worker_priorities CHECK (((array_ndims(worker_priorities) = 1) AND (array_length(worker_priorities, 1) IS NOT NULL))) -); - - --- --- Name: que_values; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.que_values ( - key text NOT NULL, - value jsonb DEFAULT '{}'::jsonb NOT NULL, - CONSTRAINT valid_value CHECK ((jsonb_typeof(value) = 'object'::text)) -) -WITH (fillfactor='90'); - - -- -- Name: quick_tips; Type: TABLE; Schema: public; Owner: - -- @@ -1598,13 +1342,6 @@ ALTER TABLE ONLY public.pages ALTER COLUMN id SET DEFAULT nextval('public.pages_ ALTER TABLE ONLY public.pg_search_documents ALTER COLUMN id SET DEFAULT nextval('public.pg_search_documents_id_seq'::regclass); --- --- Name: que_jobs id; Type: DEFAULT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.que_jobs ALTER COLUMN id SET DEFAULT nextval('public.que_jobs_id_seq'::regclass); - - -- -- Name: quick_tips id; Type: DEFAULT; Schema: public; Owner: - -- @@ -1882,30 +1619,6 @@ ALTER TABLE ONLY public.pg_search_documents ADD CONSTRAINT pg_search_documents_pkey PRIMARY KEY (id); --- --- Name: que_jobs que_jobs_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.que_jobs - ADD CONSTRAINT que_jobs_pkey PRIMARY KEY (id); - - --- --- Name: que_lockers que_lockers_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.que_lockers - ADD CONSTRAINT que_lockers_pkey PRIMARY KEY (pid); - - --- --- Name: que_values que_values_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.que_values - ADD CONSTRAINT que_values_pkey PRIMARY KEY (key); - - -- -- Name: quick_tips quick_tips_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -2331,34 +2044,6 @@ CREATE UNIQUE INDEX index_users_on_email ON public.users USING btree (email); CREATE UNIQUE INDEX index_users_on_email_lower_unique ON public.users USING btree (lower(email)); --- --- Name: que_jobs_args_gin_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX que_jobs_args_gin_idx ON public.que_jobs USING gin (args jsonb_path_ops); - - --- --- Name: que_jobs_data_gin_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX que_jobs_data_gin_idx ON public.que_jobs USING gin (data jsonb_path_ops); - - --- --- Name: que_jobs_kwargs_gin_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX que_jobs_kwargs_gin_idx ON public.que_jobs USING gin (kwargs jsonb_path_ops); - - --- --- Name: que_poll_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX que_poll_idx ON public.que_jobs USING btree (job_schema_version, queue, priority, run_at, id) WHERE ((finished_at IS NULL) AND (expired_at IS NULL)); - - -- -- Name: index_upvs.submissions_on_anonymous_user_uuid_and_uuid; Type: INDEX; Schema: upvs; Owner: - -- @@ -2387,20 +2072,6 @@ CREATE INDEX "index_upvs.submissions_on_user_id" ON upvs.submissions USING btree CREATE UNIQUE INDEX "index_upvs.submissions_on_user_id_and_uuid" ON upvs.submissions USING btree (user_id, uuid); --- --- Name: que_jobs que_job_notify; Type: TRIGGER; Schema: public; Owner: - --- - -CREATE TRIGGER que_job_notify AFTER INSERT ON public.que_jobs FOR EACH ROW WHEN ((NOT (COALESCE(current_setting('que.skip_notify'::text, true), ''::text) = 'true'::text))) EXECUTE FUNCTION public.que_job_notify(); - - --- --- Name: que_jobs que_state_notify; Type: TRIGGER; Schema: public; Owner: - --- - -CREATE TRIGGER que_state_notify AFTER INSERT OR DELETE OR UPDATE ON public.que_jobs FOR EACH ROW WHEN ((NOT (COALESCE(current_setting('que.skip_notify'::text, true), ''::text) = 'true'::text))) EXECUTE FUNCTION public.que_state_notify(); - - -- -- Name: pg_search_documents tsv_keywords_update; Type: TRIGGER; Schema: public; Owner: - -- @@ -2665,6 +2336,7 @@ INSERT INTO "schema_migrations" (version) VALUES ('20231007083404'), ('20231007083405'), ('20231007083406'), -('20231007141039'); +('20231007141039'), +('20231007153003');