From 9e99437383cbf69f01f8f46ffccd5917edb665c0 Mon Sep 17 00:00:00 2001 From: Zac Stewart Date: Sun, 3 Oct 2021 10:58:28 +0800 Subject: [PATCH 1/4] Capture initial db state as migration Originally used DM to auto-migrate, and then started using db-migrate on top of that without capturing the schema as an initial migration. Now you should be able to rake db:create db:migrate. --- db/migrate/000_starting_point.rb | 130 +++++++++++++++++++++++++++++++ db/schema.sql | 8 +- 2 files changed, 134 insertions(+), 4 deletions(-) create mode 100644 db/migrate/000_starting_point.rb diff --git a/db/migrate/000_starting_point.rb b/db/migrate/000_starting_point.rb new file mode 100644 index 0000000..a45f3b0 --- /dev/null +++ b/db/migrate/000_starting_point.rb @@ -0,0 +1,130 @@ +# Prior to this migration, databases were created and auto-upgraded by +# datamapper +STARTING_POINT = <<-SQL +-- +-- PostgreSQL database dump +-- + +-- Dumped from database version 13.4 +-- Dumped by pg_dump version 13.4 + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SELECT pg_catalog.set_config('search_path', '', false); +SET check_function_bodies = false; +SET xmloption = content; +SET client_min_messages = warning; +SET row_security = off; + +SET default_tablespace = ''; + +SET default_table_access_method = heap; + +-- +-- Name: tech404_index_channels; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.tech404_index_channels ( + id character varying(50) NOT NULL, + name character varying(50), + creator_id character varying(50), + member boolean +); + + +-- +-- Name: tech404_index_messages; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.tech404_index_messages ( + id integer NOT NULL, + channel_id character varying(50), + user_id character varying(50), + text text, + "timestamp" timestamp without time zone +); + + +-- +-- Name: tech404_index_messages_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.tech404_index_messages_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: tech404_index_messages_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.tech404_index_messages_id_seq OWNED BY public.tech404_index_messages.id; + + +-- +-- Name: tech404_index_users; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.tech404_index_users ( + id character varying(50) NOT NULL, + name character varying(50), + real_name character varying(50), + image character varying(255) +); + + +-- +-- Name: tech404_index_messages id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tech404_index_messages ALTER COLUMN id SET DEFAULT nextval('public.tech404_index_messages_id_seq'::regclass); + + +-- +-- Name: tech404_index_channels tech404_index_channels_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tech404_index_channels + ADD CONSTRAINT tech404_index_channels_pkey PRIMARY KEY (id); + + +-- +-- Name: tech404_index_messages tech404_index_messages_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tech404_index_messages + ADD CONSTRAINT tech404_index_messages_pkey PRIMARY KEY (id); + + +-- +-- Name: tech404_index_users tech404_index_users_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tech404_index_users + ADD CONSTRAINT tech404_index_users_pkey PRIMARY KEY (id); + + +-- +-- PostgreSQL database dump complete +-- + +SQL + +migration 0, :starting_point do + up do + execute STARTING_POINT + end + + down do + drop_table :tech404logs_channels + drop table :tech404logs_messages + drop_table :tech404logs_users + end +end diff --git a/db/schema.sql b/db/schema.sql index ed25055..af3eff5 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -2,8 +2,8 @@ -- PostgreSQL database dump -- --- Dumped from database version 11.4 --- Dumped by pg_dump version 11.4 +-- Dumped from database version 13.4 +-- Dumped by pg_dump version 13.4 SET statement_timeout = 0; SET lock_timeout = 0; @@ -18,7 +18,7 @@ SET row_security = off; SET default_tablespace = ''; -SET default_with_oids = false; +SET default_table_access_method = heap; -- -- Name: channels; Type: TABLE; Schema: public; Owner: - @@ -39,7 +39,6 @@ CREATE TABLE public.channels ( CREATE TABLE public.messages ( id integer NOT NULL, channel_id character varying(50), - team_id character varying(50), user_id character varying(50), text text, "timestamp" timestamp without time zone @@ -60,6 +59,7 @@ CREATE TABLE public.migration_info ( -- CREATE SEQUENCE public.tech404_index_messages_id_seq + AS integer START WITH 1 INCREMENT BY 1 NO MINVALUE From 3bf598e3cfe2c539bdd2e2550acb4a5c4ac93f02 Mon Sep 17 00:00:00 2001 From: Zac Stewart Date: Sun, 3 Oct 2021 11:53:08 +0800 Subject: [PATCH 2/4] Use a materialized view instead of building tsvec at request time --- .../005_add_fulltext_index_to_messages.rb | 25 +++++++++++++++++++ lib/tech404logs.rb | 1 + lib/tech404logs/application.rb | 8 +++--- lib/tech404logs/searchable_message.rb | 15 +++++++++++ 4 files changed, 45 insertions(+), 4 deletions(-) create mode 100644 db/migrate/005_add_fulltext_index_to_messages.rb create mode 100644 lib/tech404logs/searchable_message.rb diff --git a/db/migrate/005_add_fulltext_index_to_messages.rb b/db/migrate/005_add_fulltext_index_to_messages.rb new file mode 100644 index 0000000..523bbb7 --- /dev/null +++ b/db/migrate/005_add_fulltext_index_to_messages.rb @@ -0,0 +1,25 @@ +migration 5, :add_fulltext_index_to_messages do + up do + execute <<~SQL + CREATE MATERIALIZED VIEW searchable_messages AS + SELECT + messages.*, + channels.name AS channel_name, + users.image AS user_image, + users.real_name AS user_real_name, + users.name AS user_name, + to_tsvector(messages.text || ' ' || channels.name || ' ' || users.real_name || ' ' || users.name) AS tsv + FROM messages + INNER JOIN users ON messages.user_id = users.id + INNER JOIN channels ON messages.channel_id = channels.id + WITH NO DATA; + SQL + + execute 'CREATE INDEX searchable_messages_tsv ON searchable_messages USING gin(tsv)' + end + + down do + execute 'DROP INDEX searchable_messages_tsv' + execute 'DROP MATERIALIZED VIEW searchable_messages;' + end +end diff --git a/lib/tech404logs.rb b/lib/tech404logs.rb index 0cb4469..514a611 100644 --- a/lib/tech404logs.rb +++ b/lib/tech404logs.rb @@ -22,6 +22,7 @@ require 'tech404logs/handlers/message_handler' require 'tech404logs/link_format_filter' require 'tech404logs/message' +require 'tech404logs/searchable_message' require 'tech404logs/message_format_filter' require 'tech404logs/user' require 'tech404logs/user_mention_filter' diff --git a/lib/tech404logs/application.rb b/lib/tech404logs/application.rb index 8d906d8..de454ea 100644 --- a/lib/tech404logs/application.rb +++ b/lib/tech404logs/application.rb @@ -1,7 +1,7 @@ module Tech404logs class Application < Sinatra::Base - FULLTEXT = "to_tsvector('english', text || ' ' || channels.name || ' ' || users.name) @@ plainto_tsquery(?)".freeze + FULLTEXT = "tsv @@ plainto_tsquery(?)".freeze HOME_CHANNEL = ENV.fetch('HOME_CHANNEL').freeze helpers do @@ -54,10 +54,10 @@ class Application < Sinatra::Base content_type :html cache(request.fullpath) do - @messages = Message.all( + @messages = SearchableMessage.all( # Trick datamapper into making joins - Message.channel.id.gt => 0, - Message.user.id.gt => 0, + SearchableMessage.channel.id.gt => 0, + SearchableMessage.user.id.gt => 0, conditions: [FULLTEXT, params[:q]], limit: 100 ) diff --git a/lib/tech404logs/searchable_message.rb b/lib/tech404logs/searchable_message.rb new file mode 100644 index 0000000..0a9764c --- /dev/null +++ b/lib/tech404logs/searchable_message.rb @@ -0,0 +1,15 @@ +module Tech404logs + class SearchableMessage < Message + include DataMapper::Resource + storage_names[:default] = 'searchable_messages' + + property :id, Serial + property :channel_id, String + property :user_id, String + property :text, Text + property :timestamp, DateTime + + belongs_to :user + belongs_to :channel + end +end From 93f9074866a43ab4f45aaff63818920ce6118495 Mon Sep 17 00:00:00 2001 From: Zac Stewart Date: Sun, 3 Oct 2021 12:04:16 +0800 Subject: [PATCH 3/4] Add task to refresh search index --- Rakefile | 4 ++++ lib/tech404logs.rb | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/Rakefile b/Rakefile index 15afd69..211ac9b 100644 --- a/Rakefile +++ b/Rakefile @@ -21,6 +21,10 @@ task :environment do Tech404logs.preboot end +task :reindex => [:environment] do + Tech404logs.db.execute 'REFRESH MATERIALIZED VIEW searchable_messages' +end + namespace :db do task auto: :environment do DataMapper.auto_upgrade! diff --git a/lib/tech404logs.rb b/lib/tech404logs.rb index 514a611..aa3b1dc 100644 --- a/lib/tech404logs.rb +++ b/lib/tech404logs.rb @@ -65,6 +65,11 @@ def self.preboot Sequel.connect(ENV['DATABASE_URL'], logger: logger) end + def self.db + @db + end + + def self.production? environment == 'production' end From 7bc4393c7a3c68602eaae89a3d5d2558543de991 Mon Sep 17 00:00:00 2001 From: Zac Stewart Date: Sun, 3 Oct 2021 13:45:46 +0800 Subject: [PATCH 4/4] Okay, subclassing a model is a bad idea --- lib/tech404logs/searchable_message.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tech404logs/searchable_message.rb b/lib/tech404logs/searchable_message.rb index 0a9764c..f45af18 100644 --- a/lib/tech404logs/searchable_message.rb +++ b/lib/tech404logs/searchable_message.rb @@ -1,5 +1,5 @@ module Tech404logs - class SearchableMessage < Message + class SearchableMessage include DataMapper::Resource storage_names[:default] = 'searchable_messages'