Merge pull request #40 from zacstewart/fix-fulltext-timeout

Fix fulltext timeout

Rakefile

@@ -21,6 +21,10 @@ task :environment do
   Tech404logs.preboot
 end
 
+task :reindex => [:environment] do
+  Tech404logs.db.execute 'REFRESH MATERIALIZED VIEW searchable_messages'
+end
+
 namespace :db do
   task auto: :environment do
     DataMapper.auto_upgrade!

db/migrate/000_starting_point.rb

@@ -0,0 +1,130 @@
+# Prior to this migration, databases were created and auto-upgraded by
+# datamapper
+STARTING_POINT = <<-SQL
+--
+-- PostgreSQL database dump
+--
+
+-- Dumped from database version 13.4
+-- Dumped by pg_dump version 13.4
+
+SET statement_timeout = 0;
+SET lock_timeout = 0;
+SET idle_in_transaction_session_timeout = 0;
+SET client_encoding = 'UTF8';
+SET standard_conforming_strings = on;
+SELECT pg_catalog.set_config('search_path', '', false);
+SET check_function_bodies = false;
+SET xmloption = content;
+SET client_min_messages = warning;
+SET row_security = off;
+
+SET default_tablespace = '';
+
+SET default_table_access_method = heap;
+
+--
+-- Name: tech404_index_channels; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.tech404_index_channels (
+    id character varying(50) NOT NULL,
+    name character varying(50),
+    creator_id character varying(50),
+    member boolean
+);
+
+
+--
+-- Name: tech404_index_messages; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.tech404_index_messages (
+    id integer NOT NULL,
+    channel_id character varying(50),
+    user_id character varying(50),
+    text text,
+    "timestamp" timestamp without time zone
+);
+
+
+--
+-- Name: tech404_index_messages_id_seq; Type: SEQUENCE; Schema: public; Owner: -
+--
+
+CREATE SEQUENCE public.tech404_index_messages_id_seq
+    AS integer
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+--
+-- Name: tech404_index_messages_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
+--
+
+ALTER SEQUENCE public.tech404_index_messages_id_seq OWNED BY public.tech404_index_messages.id;
+
+
+--
+-- Name: tech404_index_users; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.tech404_index_users (
+    id character varying(50) NOT NULL,
+    name character varying(50),
+    real_name character varying(50),
+    image character varying(255)
+);
+
+
+--
+-- Name: tech404_index_messages id; Type: DEFAULT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.tech404_index_messages ALTER COLUMN id SET DEFAULT nextval('public.tech404_index_messages_id_seq'::regclass);
+
+
+--
+-- Name: tech404_index_channels tech404_index_channels_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.tech404_index_channels
+    ADD CONSTRAINT tech404_index_channels_pkey PRIMARY KEY (id);
+
+
+--
+-- Name: tech404_index_messages tech404_index_messages_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.tech404_index_messages
+    ADD CONSTRAINT tech404_index_messages_pkey PRIMARY KEY (id);
+
+
+--
+-- Name: tech404_index_users tech404_index_users_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.tech404_index_users
+    ADD CONSTRAINT tech404_index_users_pkey PRIMARY KEY (id);
+
+
+--
+-- PostgreSQL database dump complete
+--
+
+SQL
+
+migration 0, :starting_point do
+  up do
+    execute STARTING_POINT
+  end
+
+  down do
+    drop_table :tech404logs_channels
+    drop table :tech404logs_messages
+    drop_table :tech404logs_users
+  end
+end

db/migrate/005_add_fulltext_index_to_messages.rb

@@ -0,0 +1,25 @@
+migration 5, :add_fulltext_index_to_messages do
+  up do
+    execute <<~SQL
+      CREATE MATERIALIZED VIEW searchable_messages AS
+        SELECT
+          messages.*,
+          channels.name AS channel_name,
+          users.image AS user_image,
+          users.real_name AS user_real_name,
+          users.name AS user_name,
+          to_tsvector(messages.text || ' ' || channels.name || ' ' || users.real_name || ' ' || users.name) AS tsv
+        FROM messages
+        INNER JOIN users ON messages.user_id = users.id
+        INNER JOIN channels ON messages.channel_id = channels.id
+      WITH NO DATA;
+    SQL
+
+    execute 'CREATE INDEX searchable_messages_tsv ON searchable_messages USING gin(tsv)'
+  end
+
+  down do
+    execute 'DROP INDEX searchable_messages_tsv'
+    execute 'DROP MATERIALIZED VIEW searchable_messages;'
+  end
+end

db/schema.sql

@@ -2,8 +2,8 @@
 -- PostgreSQL database dump
 --
 
--- Dumped from database version 11.4
--- Dumped by pg_dump version 11.4
+-- Dumped from database version 13.4
+-- Dumped by pg_dump version 13.4
 
 SET statement_timeout = 0;
 SET lock_timeout = 0;
@@ -18,7 +18,7 @@ SET row_security = off;
 
 SET default_tablespace = '';
 
-SET default_with_oids = false;
+SET default_table_access_method = heap;
 
 --
 -- Name: channels; Type: TABLE; Schema: public; Owner: -
@@ -39,7 +39,6 @@ CREATE TABLE public.channels (
 CREATE TABLE public.messages (
     id integer NOT NULL,
     channel_id character varying(50),
-    team_id character varying(50),
     user_id character varying(50),
     text text,
     "timestamp" timestamp without time zone
@@ -60,6 +59,7 @@ CREATE TABLE public.migration_info (
 --
 
 CREATE SEQUENCE public.tech404_index_messages_id_seq
+    AS integer
     START WITH 1
     INCREMENT BY 1
     NO MINVALUE

lib/tech404logs.rb

@@ -22,6 +22,7 @@
 require 'tech404logs/handlers/message_handler'
 require 'tech404logs/link_format_filter'
 require 'tech404logs/message'
+require 'tech404logs/searchable_message'
 require 'tech404logs/message_format_filter'
 require 'tech404logs/user'
 require 'tech404logs/user_mention_filter'
@@ -64,6 +65,11 @@ def self.preboot
       Sequel.connect(ENV['DATABASE_URL'], logger: logger)
   end
 
+  def self.db
+    @db
+  end
+
+
   def self.production?
     environment == 'production'
   end

lib/tech404logs/application.rb

@@ -1,7 +1,7 @@
 module Tech404logs
   class Application < Sinatra::Base
 
-    FULLTEXT = "to_tsvector('english', text || ' ' || channels.name || ' ' || users.name) @@ plainto_tsquery(?)".freeze
+    FULLTEXT = "tsv @@ plainto_tsquery(?)".freeze
     HOME_CHANNEL = ENV.fetch('HOME_CHANNEL').freeze
 
     helpers do
@@ -54,10 +54,10 @@ class Application < Sinatra::Base
       content_type :html
 
       cache(request.fullpath) do
-        @messages = Message.all(
+        @messages = SearchableMessage.all(
           # Trick datamapper into making joins
-          Message.channel.id.gt => 0,
-          Message.user.id.gt => 0,
+          SearchableMessage.channel.id.gt => 0,
+          SearchableMessage.user.id.gt => 0,
           conditions: [FULLTEXT, params[:q]],
           limit: 100
         )

lib/tech404logs/searchable_message.rb

@@ -0,0 +1,15 @@
+module Tech404logs
+  class SearchableMessage
+    include DataMapper::Resource
+    storage_names[:default] = 'searchable_messages'
+
+    property :id, Serial
+    property :channel_id, String
+    property :user_id, String
+    property :text, Text
+    property :timestamp, DateTime
+
+    belongs_to :user
+    belongs_to :channel
+  end
+end