From c993d9dfdd7cf241de8b068ab4ce616e8292131c Mon Sep 17 00:00:00 2001 From: Karl Ostmo Date: Sun, 12 May 2024 13:45:08 -0700 Subject: [PATCH] Use sqlite and static binary (#1837) This is a rework of #1798 to facilitate a simpler web stack. # Demo View http://swarmgame.net/ NOTE: Requires IPv6 # Motivation Hosting cost is a main motivation. Cost per month for an EC2 instance, RDS, and the requisite other services approaches >$50 per month. In contrast, the lowest-tier Lightsail instance is $3.50/month. The deployment process is of course simplified. An incidental benefit to using SQLite is reduced latency of web requests; we no longer need to fetch credentials from an AWS API to connect to Postgres. ## Changes Major changes: * Use `sqlite` instead of `postgres` * Use Docker to build a statically-linked deployable binary, rather than deploying the app within a Docker image Fortunately, the API of `sqlite-simple` is near-identical to that of `postgresql-simple`, so most of the code change there is just to rip out AWS-specific stuff and Postgres connection info. I have no hesitation to delete this code since if we ever want to use the previous stack again, we can just look at #1798. --- app/tournament/Main.hs | 27 +- scripts/test/run-tests.sh | 6 +- src/swarm-tournament/Swarm/Web/Tournament.hs | 23 +- .../Swarm/Web/Tournament/Database/Query.hs | 112 +------- .../Swarm/Web/Tournament/Type.hs | 2 +- swarm.cabal | 5 +- test/tournament-host/Main.hs | 2 - tournament/README.md | 23 ++ tournament/schema/schema-local.sql | 267 ------------------ tournament/schema/swarm-sqlite-schema.sql | 54 ++++ .../scripts/database/dump-local-schema.sh | 2 +- .../database/recreate-local-database.sh | 5 +- tournament/scripts/demo/README.md | 22 +- .../client/test-cases/local/good-submit.sh | 4 +- .../local/wrong-scenario-cached-solution.sh | 6 +- tournament/scripts/demo/server-docker.sh | 14 - tournament/scripts/demo/server-native.sh | 3 +- tournament/scripts/deploy/redeploy-binary.sh | 17 ++ .../scripts/deploy/redeploy-web-files.sh | 6 + tournament/scripts/docker/Dockerfile | 101 ------- tournament/scripts/docker/alpine/Dockerfile | 57 ++++ tournament/scripts/docker/aws-login.sh | 5 - tournament/scripts/docker/build-image.sh | 11 - .../scripts/docker/build-server-executable.sh | 11 +- .../scripts/docker/build-static-binary.sh | 14 + tournament/scripts/docker/docker-prereqs.sh | 6 - .../scripts/docker/local-docker-shell.sh | 6 - .../scripts/docker/local-pg-credentials.env | 1 - tournament/scripts/docker/redeploy-image.sh | 16 -- 29 files changed, 220 insertions(+), 608 deletions(-) create mode 100644 tournament/README.md delete mode 100644 tournament/schema/schema-local.sql create mode 100644 tournament/schema/swarm-sqlite-schema.sql delete mode 100755 tournament/scripts/demo/server-docker.sh create mode 100755 tournament/scripts/deploy/redeploy-binary.sh create mode 100755 tournament/scripts/deploy/redeploy-web-files.sh delete mode 100644 tournament/scripts/docker/Dockerfile create mode 100644 tournament/scripts/docker/alpine/Dockerfile delete mode 100755 tournament/scripts/docker/aws-login.sh delete mode 100755 tournament/scripts/docker/build-image.sh create mode 100755 tournament/scripts/docker/build-static-binary.sh delete mode 100644 tournament/scripts/docker/docker-prereqs.sh delete mode 100755 tournament/scripts/docker/local-docker-shell.sh delete mode 100644 tournament/scripts/docker/local-pg-credentials.env delete mode 100755 tournament/scripts/docker/redeploy-image.sh diff --git a/app/tournament/Main.hs b/app/tournament/Main.hs index 9d2fabb9a..2c91fcb64 100644 --- a/app/tournament/Main.hs +++ b/app/tournament/Main.hs @@ -3,15 +3,12 @@ module Main where import Control.Monad.Trans.Reader (runReaderT) -import Data.IORef (newIORef) import Data.Maybe (fromMaybe) import Network.Wai.Handler.Warp (Port) import Options.Applicative import Swarm.Game.State (Sha1 (..)) import Swarm.Web.Tournament import Swarm.Web.Tournament.Database.Query -import System.Environment (lookupEnv) -import System.Posix.User (getEffectiveUserName) data AppOpts = AppOpts { userWebPort :: Maybe Port @@ -57,25 +54,14 @@ cliInfo = <> fullDesc ) -deduceConnType :: Bool -> IO DbConnType -deduceConnType isLocalSocketConn = - if isLocalSocketConn - then LocalDBOverSocket . Username <$> getEffectiveUserName - else do - maybeDbPassword <- lookupEnv envarPostgresPasswordKey - case maybeDbPassword of - Just dbPasswordEnvar -> return $ LocalDBFromDockerOverNetwork $ Password dbPasswordEnvar - Nothing -> RemoteDB <$> newIORef Nothing - main :: IO () main = do opts <- execParser cliInfo - connType <- deduceConnType $ isLocalSocketConnection opts webMain - (AppData (gameGitVersion opts) (persistenceFunctions connType) connType) + (AppData (gameGitVersion opts) persistenceFunctions) (fromMaybe defaultPort $ userWebPort opts) where - persistenceFunctions connMode = + persistenceFunctions = PersistenceLayer { lookupScenarioFileContent = withConnInfo lookupScenarioContent , scenarioStorage = @@ -90,10 +76,5 @@ main = do } } where - withConnInfo f x = do - -- This gets deferred and re-executed upon each invocation - -- of a DB interaction function. - -- We need this behavior because the password fetched via API - -- expires after 15 min. - connInfo <- mkConnectInfo connMode - runReaderT (f x) connInfo + withConnInfo f x = + runReaderT (f x) databaseFilename diff --git a/scripts/test/run-tests.sh b/scripts/test/run-tests.sh index 24e86dd58..6de8cbea2 100755 --- a/scripts/test/run-tests.sh +++ b/scripts/test/run-tests.sh @@ -1,7 +1,5 @@ #!/bin/bash -ex -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -cd $SCRIPT_DIR/.. +cd $(git rev-parse --show-toplevel) -# See https://github.com/swarm-game/swarm/issues/936 -STACK_WORK=.stack-work-test stack test --fast "$@" +cabal test --test-show-details=direct -O0 -j "$@" diff --git a/src/swarm-tournament/Swarm/Web/Tournament.hs b/src/swarm-tournament/Swarm/Web/Tournament.hs index 41cbfa529..f64cae09f 100644 --- a/src/swarm-tournament/Swarm/Web/Tournament.hs +++ b/src/swarm-tournament/Swarm/Web/Tournament.hs @@ -71,7 +71,6 @@ defaultSolutionTimeout = SolutionTimeout 15 data AppData = AppData { swarmGameGitVersion :: Sha1 , persistence :: PersistenceLayer - , dbConnType :: DbConnType } type TournamentAPI = @@ -126,10 +125,10 @@ mkApp appData = :<|> uploadSolution appData :<|> getScenarioMetadata appData :<|> downloadRedactedScenario appData - :<|> listScenarios appData + :<|> listScenarios uploadScenario :: AppData -> MultipartData Mem -> Handler ScenarioCharacterization -uploadScenario (AppData gameVersion persistenceLayer _) multipartData = +uploadScenario (AppData gameVersion persistenceLayer) multipartData = Handler . withExceptT toServantError . ExceptT $ validateScenarioUpload args @@ -144,7 +143,7 @@ uploadScenario (AppData gameVersion persistenceLayer _) multipartData = (scenarioStorage persistenceLayer) uploadSolution :: AppData -> MultipartData Mem -> Handler SolutionFileCharacterization -uploadSolution (AppData _ persistenceLayer _) multipartData = +uploadSolution (AppData _ persistenceLayer) multipartData = Handler . withExceptT toServantError . ExceptT $ validateSubmittedSolution args @@ -159,7 +158,7 @@ uploadSolution (AppData _ persistenceLayer _) multipartData = (solutionStorage persistenceLayer) getScenarioMetadata :: AppData -> Sha1 -> Handler ScenarioMetadata -getScenarioMetadata (AppData _ persistenceLayer _) scenarioSha1 = +getScenarioMetadata (AppData _ persistenceLayer) scenarioSha1 = Handler . withExceptT toServantError $ do doc <- ExceptT $ @@ -170,7 +169,7 @@ getScenarioMetadata (AppData _ persistenceLayer _) scenarioSha1 = return $ view scenarioMetadata s downloadRedactedScenario :: AppData -> Sha1 -> Handler TL.Text -downloadRedactedScenario (AppData _ persistenceLayer _) scenarioSha1 = do +downloadRedactedScenario (AppData _ persistenceLayer) scenarioSha1 = do Handler . withExceptT toServantError $ do doc <- ExceptT $ @@ -183,12 +182,12 @@ downloadRedactedScenario (AppData _ persistenceLayer _) scenarioSha1 = do encodeWith defaultEncodeOptions redactedDict -- NOTE: This is currently the only API endpoint that invokes --- 'mkConnectInfo' directly -listScenarios :: AppData -> Handler [TournamentGame] -listScenarios (AppData _ _ connMode) = - Handler $ liftIO $ do - connInfo <- mkConnectInfo connMode - runReaderT listGames connInfo +-- 'runReaderT' directly +listScenarios :: Handler [TournamentGame] +listScenarios = + Handler $ + liftIO $ + runReaderT listGames databaseFilename -- * Web app declaration diff --git a/src/swarm-tournament/Swarm/Web/Tournament/Database/Query.hs b/src/swarm-tournament/Swarm/Web/Tournament/Database/Query.hs index cbbb7a871..fba974dd3 100644 --- a/src/swarm-tournament/Swarm/Web/Tournament/Database/Query.hs +++ b/src/swarm-tournament/Swarm/Web/Tournament/Database/Query.hs @@ -10,28 +10,24 @@ -- SQL Queries for Swarm tournaments. module Swarm.Web.Tournament.Database.Query where -import Control.Monad (guard) import Control.Monad.IO.Class (liftIO) import Control.Monad.Trans.Maybe (MaybeT (..), runMaybeT) import Control.Monad.Trans.Reader (ReaderT, ask) import Data.ByteString.Lazy qualified as LBS import Data.IORef import Data.Maybe (listToMaybe) -import Data.String.Utils (strip) import Data.Time.Clock -import Database.PostgreSQL.Simple -import Database.PostgreSQL.Simple.FromRow -import Database.PostgreSQL.Simple.ToField +import Database.SQLite.Simple +import Database.SQLite.Simple.ToField import Swarm.Game.Scenario.Scoring.CodeSize import Swarm.Game.State (Sha1 (..)) import Swarm.Game.Tick (TickNumber (..)) import Swarm.Web.Tournament.Type -import System.Exit (ExitCode (..)) -import System.Process --- | Used for local development only -envarPostgresPasswordKey :: String -envarPostgresPasswordKey = "LOCAL_PGPASS" +type ConnectInfo = String + +databaseFilename :: ConnectInfo +databaseFilename = "swarm-games.db" newtype UserId = UserId Int @@ -117,90 +113,6 @@ data DbConnType tokenRefreshInterval :: NominalDiffTime tokenRefreshInterval = 10 * 60 -genNewToken :: ConnectInfo -> IO (Either String String) -genNewToken ci = do - (exitCode, stdoutString, stderrString) <- - readProcessWithExitCode - "aws" - [ "rds" - , "generate-db-auth-token" - , "--hostname" - , connectHost ci - , "--port" - , show $ connectPort ci - , "--region" - , region - , "--username" - , connectUser ci - ] - "" - return $ case exitCode of - ExitSuccess -> Right $ strip stdoutString - ExitFailure _ -> Left stderrString - where - region = "us-east-1" - -getAwsCredentials :: TokenRef -> ConnectInfo -> IO ConnectInfo -getAwsCredentials tokRef ci = do - currTime <- getCurrentTime - maybePreviousTok <- readIORef tokRef - let maybeStillValidTok = case maybePreviousTok of - Nothing -> Nothing - Just (TokenWithExpiration exprTime tok) -> - guard (currTime < exprTime) >> Just tok - - case maybeStillValidTok of - Just (Password tok) -> - return $ - ci - { connectPassword = tok - } - Nothing -> do - eitherNewTok <- genNewToken ci - case eitherNewTok of - Right newTok -> do - let nextExpirationTime = addUTCTime tokenRefreshInterval currTime - atomicWriteIORef tokRef - . Just - . TokenWithExpiration nextExpirationTime - $ Password newTok - return $ - ci - { connectPassword = newTok - } - -- NOTE: This is not exactly valid behavior: - Left _errMsg -> return ci - -mkConnectInfo :: DbConnType -> IO ConnectInfo -mkConnectInfo connType = do - let swarmDbConnect = - defaultConnectInfo - { connectDatabase = "swarm" - } - - case connType of - LocalDBFromDockerOverNetwork (Password dbPasswd) -> - return $ - swarmDbConnect - { connectHost = "host.docker.internal" - , connectUser = "swarm-app" - , connectPassword = dbPasswd - } - LocalDBOverSocket (Username username) -> - return - swarmDbConnect - { connectHost = "/var/run/postgresql" - , connectUser = username - } - RemoteDB tokRef -> getAwsCredentials tokRef rdsConnectionInfo - where - rdsConnectionInfo = - defaultConnectInfo - { connectHost = "swarm-tournaments.cv6iymakujnb.us-east-1.rds.amazonaws.com" - , connectUser = "swarm-app" - , connectDatabase = "swarm" - } - -- * Authentication getUserId :: Connection -> UserAlias -> IO UserId @@ -226,13 +138,13 @@ getUserId conn userAlias = do lookupScenarioContent :: Sha1 -> ReaderT ConnectInfo IO (Maybe LBS.ByteString) lookupScenarioContent sha1 = do connInfo <- ask - liftIO . fmap (fmap fromOnly . listToMaybe) . withConnect connInfo $ \conn -> + liftIO . fmap (fmap fromOnly . listToMaybe) . withConnection connInfo $ \conn -> query conn "SELECT content FROM scenarios WHERE content_sha1 = ?;" (Only sha1) lookupSolutionSubmission :: Sha1 -> ReaderT ConnectInfo IO (Maybe AssociatedSolutionSolutionCharacterization) lookupSolutionSubmission contentSha1 = do connInfo <- ask - liftIO $ withConnect connInfo $ \conn -> runMaybeT $ do + liftIO $ withConnection connInfo $ \conn -> runMaybeT $ do evaluationId :: Int <- MaybeT $ fmap fromOnly . listToMaybe @@ -246,14 +158,14 @@ lookupSolutionSubmission contentSha1 = do lookupScenarioSolution :: Sha1 -> ReaderT ConnectInfo IO (Maybe AssociatedSolutionSolutionCharacterization) lookupScenarioSolution scenarioSha1 = do connInfo <- ask - solnChar <- liftIO . fmap listToMaybe . withConnect connInfo $ \conn -> + solnChar <- liftIO . fmap listToMaybe . withConnection connInfo $ \conn -> query conn "SELECT wall_time_seconds, ticks, seed, char_count, ast_size FROM evaluated_solution WHERE builtin AND scenario = ? LIMIT 1;" (Only scenarioSha1) return $ AssociatedSolutionSolutionCharacterization scenarioSha1 <$> solnChar listGames :: ReaderT ConnectInfo IO [TournamentGame] listGames = do connInfo <- ask - liftIO $ withConnect connInfo $ \conn -> + liftIO $ withConnection connInfo $ \conn -> query_ conn "SELECT original_filename, scenario_uploader, scenario, submission_count, swarm_git_sha1 FROM submissions;" -- * Insertion @@ -263,7 +175,7 @@ insertScenario :: ReaderT ConnectInfo IO Sha1 insertScenario s = do connInfo <- ask - h <- liftIO $ withConnect connInfo $ \conn -> do + h <- liftIO $ withConnection connInfo $ \conn -> do uid <- getUserId conn $ uploader $ upload s [Only resultList] <- query @@ -287,7 +199,7 @@ insertSolutionSubmission :: ReaderT ConnectInfo IO Sha1 insertSolutionSubmission (CharacterizationResponse solutionUpload s (SolutionUploadResponsePayload scenarioSha)) = do connInfo <- ask - liftIO $ withConnect connInfo $ \conn -> do + liftIO $ withConnection connInfo $ \conn -> do uid <- getUserId conn $ uploader solutionUpload solutionEvalId <- insertSolution conn False scenarioSha $ characterization s diff --git a/src/swarm-tournament/Swarm/Web/Tournament/Type.hs b/src/swarm-tournament/Swarm/Web/Tournament/Type.hs index 8ffdf6cd0..0066e59cb 100644 --- a/src/swarm-tournament/Swarm/Web/Tournament/Type.hs +++ b/src/swarm-tournament/Swarm/Web/Tournament/Type.hs @@ -10,7 +10,7 @@ module Swarm.Web.Tournament.Type where import Data.Aeson import Data.ByteString.Lazy qualified as LBS import Data.Text qualified as T -import Database.PostgreSQL.Simple.ToField +import Database.SQLite.Simple.ToField import GHC.Generics (Generic) import Servant import Servant.Docs (ToCapture) diff --git a/swarm.cabal b/swarm.cabal index f72eacb46..672b959b3 100644 --- a/swarm.cabal +++ b/swarm.cabal @@ -462,7 +462,6 @@ library swarm-tournament other-modules: Paths_swarm autogen-modules: Paths_swarm build-depends: - MissingH, SHA, aeson, base, @@ -474,11 +473,10 @@ library swarm-tournament http-types, lens, mtl, - postgresql-simple >=0.7 && <0.7.1, - process, servant-docs, servant-multipart, servant-server >=0.19 && <0.21, + sqlite-simple >=0.4.19.0 && <0.4.20, text, time, transformers, @@ -757,7 +755,6 @@ executable swarm-host-tournament base, optparse-applicative >=0.16 && <0.19, transformers, - unix, warp, build-depends: diff --git a/test/tournament-host/Main.hs b/test/tournament-host/Main.hs index 6ab8bea98..282c0d239 100644 --- a/test/tournament-host/Main.hs +++ b/test/tournament-host/Main.hs @@ -50,8 +50,6 @@ main = do Tournament.AppData { Tournament.swarmGameGitVersion = Sha1 "abcdef" , Tournament.persistence = mkPersistenceLayer scenariosMap - , -- NOTE: This is not actually used/exercised by the tests: - Tournament.dbConnType = LocalDBOverSocket $ Username "" } type LocalFileLookup = NEMap Sha1 FilePathAndContent diff --git a/tournament/README.md b/tournament/README.md new file mode 100644 index 000000000..4c72bc0f7 --- /dev/null +++ b/tournament/README.md @@ -0,0 +1,23 @@ +# Usage + +## Installation prerequisites: + +Install sqlite: +``` +sudo apt install sqlite3 +``` + +## Deployment + +Run this script (requires Docker): +``` +tournament/scripts/docker/build-static-binary.sh +``` + +# Testing + +## Unit tests + +``` +scripts/test/run-tests.sh swarm:test:tournament-host +``` \ No newline at end of file diff --git a/tournament/schema/schema-local.sql b/tournament/schema/schema-local.sql deleted file mode 100644 index 8a72f1986..000000000 --- a/tournament/schema/schema-local.sql +++ /dev/null @@ -1,267 +0,0 @@ --- --- PostgreSQL database dump --- - --- Dumped from database version 14.11 (Ubuntu 14.11-0ubuntu0.22.04.1) --- Dumped by pg_dump version 14.11 (Ubuntu 14.11-0ubuntu0.22.04.1) - -SET statement_timeout = 0; -SET lock_timeout = 0; -SET idle_in_transaction_session_timeout = 0; -SET client_encoding = 'UTF8'; -SET standard_conforming_strings = on; -SELECT pg_catalog.set_config('search_path', '', false); -SET check_function_bodies = false; -SET xmloption = content; -SET client_min_messages = warning; -SET row_security = off; - --- --- Name: swarm; Type: DATABASE; Schema: -; Owner: postgres --- - -CREATE DATABASE swarm WITH TEMPLATE = template0 ENCODING = 'UTF8' LOCALE = 'en_US.UTF-8'; - - -ALTER DATABASE swarm OWNER TO postgres; - -\connect swarm - -SET statement_timeout = 0; -SET lock_timeout = 0; -SET idle_in_transaction_session_timeout = 0; -SET client_encoding = 'UTF8'; -SET standard_conforming_strings = on; -SELECT pg_catalog.set_config('search_path', '', false); -SET check_function_bodies = false; -SET xmloption = content; -SET client_min_messages = warning; -SET row_security = off; - -SET default_tablespace = ''; - -SET default_table_access_method = heap; - --- --- Name: evaluated_solution; Type: TABLE; Schema: public; Owner: kostmo --- - -CREATE TABLE public.evaluated_solution ( - id integer NOT NULL, - evaluated_at timestamp with time zone DEFAULT now() NOT NULL, - scenario character varying(40) NOT NULL, - seed bigint NOT NULL, - wall_time_seconds double precision NOT NULL, - ticks bigint, - char_count integer, - ast_size integer, - builtin boolean NOT NULL -); - - -ALTER TABLE public.evaluated_solution OWNER TO kostmo; - --- --- Name: scenarios; Type: TABLE; Schema: public; Owner: kostmo --- - -CREATE TABLE public.scenarios ( - content_sha1 character varying(40) NOT NULL, - uploader integer NOT NULL, - original_filename text, - swarm_git_sha1 character varying(40), - uploaded_at timestamp with time zone DEFAULT now() NOT NULL, - content text NOT NULL -); - - -ALTER TABLE public.scenarios OWNER TO kostmo; - --- --- Name: solution_id_seq; Type: SEQUENCE; Schema: public; Owner: kostmo --- - -CREATE SEQUENCE public.solution_id_seq - AS integer - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - - -ALTER TABLE public.solution_id_seq OWNER TO kostmo; - --- --- Name: solution_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: kostmo --- - -ALTER SEQUENCE public.solution_id_seq OWNED BY public.evaluated_solution.id; - - --- --- Name: solution_submission; Type: TABLE; Schema: public; Owner: kostmo --- - -CREATE TABLE public.solution_submission ( - content_sha1 character varying(40) NOT NULL, - uploader integer NOT NULL, - uploaded_at timestamp with time zone DEFAULT now() NOT NULL, - solution_evaluation integer -); - - -ALTER TABLE public.solution_submission OWNER TO kostmo; - --- --- Name: users; Type: TABLE; Schema: public; Owner: kostmo --- - -CREATE TABLE public.users ( - id integer NOT NULL, - alias text NOT NULL, - created_at timestamp with time zone DEFAULT now() NOT NULL -); - - -ALTER TABLE public.users OWNER TO kostmo; - --- --- Name: submissions; Type: VIEW; Schema: public; Owner: kostmo --- - -CREATE VIEW public.submissions AS - SELECT scenarios.original_filename, - scenarios.content_sha1 AS scenario, - scenarios.uploaded_at AS scenario_uploaded_at, - COALESCE(foo.submission_count, (0)::bigint) AS submission_count, - users.alias AS scenario_uploader, - scenarios.swarm_git_sha1 - FROM ((public.scenarios - LEFT JOIN ( SELECT evaluated_solution.scenario, - count(*) AS submission_count - FROM public.evaluated_solution - WHERE (NOT evaluated_solution.builtin) - GROUP BY evaluated_solution.scenario) foo ON (((scenarios.content_sha1)::text = (foo.scenario)::text))) - JOIN public.users ON ((scenarios.uploader = users.id))); - - -ALTER TABLE public.submissions OWNER TO kostmo; - --- --- Name: users_id_seq; Type: SEQUENCE; Schema: public; Owner: kostmo --- - -CREATE SEQUENCE public.users_id_seq - AS integer - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - - -ALTER TABLE public.users_id_seq OWNER TO kostmo; - --- --- Name: users_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: kostmo --- - -ALTER SEQUENCE public.users_id_seq OWNED BY public.users.id; - - --- --- Name: evaluated_solution id; Type: DEFAULT; Schema: public; Owner: kostmo --- - -ALTER TABLE ONLY public.evaluated_solution ALTER COLUMN id SET DEFAULT nextval('public.solution_id_seq'::regclass); - - --- --- Name: users id; Type: DEFAULT; Schema: public; Owner: kostmo --- - -ALTER TABLE ONLY public.users ALTER COLUMN id SET DEFAULT nextval('public.users_id_seq'::regclass); - - --- --- Name: scenarios scenarios_pkey; Type: CONSTRAINT; Schema: public; Owner: kostmo --- - -ALTER TABLE ONLY public.scenarios - ADD CONSTRAINT scenarios_pkey PRIMARY KEY (content_sha1); - - --- --- Name: solution_submission solution_file_pkey; Type: CONSTRAINT; Schema: public; Owner: kostmo --- - -ALTER TABLE ONLY public.solution_submission - ADD CONSTRAINT solution_file_pkey PRIMARY KEY (content_sha1); - - --- --- Name: evaluated_solution solution_pkey; Type: CONSTRAINT; Schema: public; Owner: kostmo --- - -ALTER TABLE ONLY public.evaluated_solution - ADD CONSTRAINT solution_pkey PRIMARY KEY (id); - - --- --- Name: users users_pkey; Type: CONSTRAINT; Schema: public; Owner: kostmo --- - -ALTER TABLE ONLY public.users - ADD CONSTRAINT users_pkey PRIMARY KEY (id); - - --- --- Name: fki_solution_file_solution; Type: INDEX; Schema: public; Owner: kostmo --- - -CREATE INDEX fki_solution_file_solution ON public.solution_submission USING btree (solution_evaluation); - - --- --- Name: fki_solution_scenario; Type: INDEX; Schema: public; Owner: kostmo --- - -CREATE INDEX fki_solution_scenario ON public.evaluated_solution USING btree (scenario); - - --- --- Name: scenario_uploader; Type: INDEX; Schema: public; Owner: kostmo --- - -CREATE INDEX scenario_uploader ON public.scenarios USING btree (uploader); - - --- --- Name: scenarios scenarios_uploader_fkey; Type: FK CONSTRAINT; Schema: public; Owner: kostmo --- - -ALTER TABLE ONLY public.scenarios - ADD CONSTRAINT scenarios_uploader_fkey FOREIGN KEY (uploader) REFERENCES public.users(id) NOT VALID; - - --- --- Name: solution_submission solution_file_solution; Type: FK CONSTRAINT; Schema: public; Owner: kostmo --- - -ALTER TABLE ONLY public.solution_submission - ADD CONSTRAINT solution_file_solution FOREIGN KEY (solution_evaluation) REFERENCES public.evaluated_solution(id) NOT VALID; - - --- --- Name: evaluated_solution solution_scenario; Type: FK CONSTRAINT; Schema: public; Owner: kostmo --- - -ALTER TABLE ONLY public.evaluated_solution - ADD CONSTRAINT solution_scenario FOREIGN KEY (scenario) REFERENCES public.scenarios(content_sha1) NOT VALID; - - --- --- PostgreSQL database dump complete --- - diff --git a/tournament/schema/swarm-sqlite-schema.sql b/tournament/schema/swarm-sqlite-schema.sql new file mode 100644 index 000000000..4415b99cf --- /dev/null +++ b/tournament/schema/swarm-sqlite-schema.sql @@ -0,0 +1,54 @@ +BEGIN TRANSACTION; +CREATE TABLE IF NOT EXISTS "users" ( + "id" INTEGER NOT NULL UNIQUE, + "alias" TEXT NOT NULL, + "created_at" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY("id" AUTOINCREMENT) +); +CREATE TABLE IF NOT EXISTS "scenarios" ( + "content_sha1" TEXT NOT NULL UNIQUE, + "uploader" INTEGER NOT NULL, + "original_filename" TEXT, + "swarm_git_sha1" TEXT, + "uploaded_at" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + "content" TEXT NOT NULL, + PRIMARY KEY("content_sha1"), + FOREIGN KEY(uploader) REFERENCES users(id) +); +CREATE TABLE IF NOT EXISTS "evaluated_solution" ( + "id" INTEGER NOT NULL UNIQUE, + "evaluated_at" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + "scenario" TEXT NOT NULL, + "seed" INTEGER NOT NULL, + "wall_time_seconds" REAL NOT NULL, + "ticks" INTEGER, + "char_count" INTEGER, + "ast_size" INTEGER, + "builtin" BOOLEAN, + PRIMARY KEY("id" AUTOINCREMENT), + FOREIGN KEY(scenario) REFERENCES scenarios(content_sha1) +); +CREATE TABLE IF NOT EXISTS "solution_submission" ( + "content_sha1" TEXT NOT NULL, + "uploader" INTEGER NOT NULL, + "uploaded_at" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + "solution_evaluation" INTEGER, + PRIMARY KEY("content_sha1"), + FOREIGN KEY(solution_evaluation) REFERENCES evaluated_solution(id) +); + +CREATE VIEW submissions AS + SELECT scenarios.original_filename, + scenarios.content_sha1 AS scenario, + scenarios.uploaded_at AS scenario_uploaded_at, + COALESCE(foo.submission_count, 0) AS submission_count, + users.alias AS scenario_uploader, + scenarios.swarm_git_sha1 + FROM ((scenarios + LEFT JOIN ( SELECT evaluated_solution.scenario, + count(*) AS submission_count + FROM evaluated_solution + WHERE (NOT evaluated_solution.builtin) + GROUP BY evaluated_solution.scenario) foo ON (scenarios.content_sha1 = foo.scenario)) + JOIN users ON (scenarios.uploader = users.id)); +COMMIT; diff --git a/tournament/scripts/database/dump-local-schema.sh b/tournament/scripts/database/dump-local-schema.sh index 375973530..a167d71c2 100755 --- a/tournament/scripts/database/dump-local-schema.sh +++ b/tournament/scripts/database/dump-local-schema.sh @@ -2,4 +2,4 @@ GIT_ROOT_DIR=$(git rev-parse --show-toplevel) -pg_dump --create -s -d swarm > $GIT_ROOT_DIR/tournament/schema/schema-local.sql +sqlite3 swarm-games.db '.schema' > $GIT_ROOT_DIR/tournament/schema/swarm-sqlite-schema.sql diff --git a/tournament/scripts/database/recreate-local-database.sh b/tournament/scripts/database/recreate-local-database.sh index aba1ebc50..80111ba36 100755 --- a/tournament/scripts/database/recreate-local-database.sh +++ b/tournament/scripts/database/recreate-local-database.sh @@ -2,7 +2,4 @@ GIT_ROOT_DIR=$(git rev-parse --show-toplevel) -sudo service postgresql restart -dropdb swarm - -sudo -u postgres psql < $GIT_ROOT_DIR/tournament/schema/schema-local.sql +sqlite3 swarm-games.db < $GIT_ROOT_DIR/tournament/schema/swarm-sqlite-schema.sql diff --git a/tournament/scripts/demo/README.md b/tournament/scripts/demo/README.md index 88eff4089..f602c3874 100644 --- a/tournament/scripts/demo/README.md +++ b/tournament/scripts/demo/README.md @@ -1,27 +1,11 @@ # Running in local development environment -The `client.sh` script can be run with either the `server-docker.sh` or the `server-native.sh` script as the host. +The `client.sh` script can be run with the `server-native.sh` script as the host. -Running the server application natively is the simplest option and connects to the local Postgres database via a socket. - -Running the server inside a local Docker image requires supplying the Postgres password as an environment variable. +Running the server application natively is the simplest option and connects to the local database file. ## Database setup -One first needs to install a local Postgres server. - -After configuring logins and users, one may populate the database using the stored `schema-local.sql` schema with a script: +One may populate the database using the committed schema with a script: tournament/scripts/database/recreate-local-database.sh - -### Configuring database access from Docker - -See this answer: https://stackoverflow.com/a/58015643/105137 - -To summarize: - -* Edit `postgresql.conf`, uncomment and set `listen_addresses = '*'` -* Edit `pg_hba.conf`, add the line: - ``` - host all all 172.17.0.0/16 password - ``` diff --git a/tournament/scripts/demo/client/test-cases/local/good-submit.sh b/tournament/scripts/demo/client/test-cases/local/good-submit.sh index 1d28a44be..a7d9bb75b 100755 --- a/tournament/scripts/demo/client/test-cases/local/good-submit.sh +++ b/tournament/scripts/demo/client/test-cases/local/good-submit.sh @@ -2,7 +2,9 @@ cd $(git rev-parse --show-toplevel) +HOST=${1:-localhost:8080} + tournament/scripts/demo/client/submit.sh \ - localhost:8008 \ + $HOST \ data/scenarios/Challenges/arbitrage.yaml \ data/scenarios/Challenges/_arbitrage/solution.sw diff --git a/tournament/scripts/demo/client/test-cases/local/wrong-scenario-cached-solution.sh b/tournament/scripts/demo/client/test-cases/local/wrong-scenario-cached-solution.sh index 1bc50530a..81e3b278e 100755 --- a/tournament/scripts/demo/client/test-cases/local/wrong-scenario-cached-solution.sh +++ b/tournament/scripts/demo/client/test-cases/local/wrong-scenario-cached-solution.sh @@ -5,9 +5,11 @@ cd $(git rev-parse --show-toplevel) -tournament/scripts/demo/client/test-cases/local/good-submit.sh +HOST=${1:-localhost:8080} + +tournament/scripts/demo/client/test-cases/local/good-submit.sh $HOST tournament/scripts/demo/client/submit.sh \ - localhost:8008 \ + $HOST \ data/scenarios/Challenges/dimsum.yaml \ data/scenarios/Challenges/_arbitrage/solution.sw diff --git a/tournament/scripts/demo/server-docker.sh b/tournament/scripts/demo/server-docker.sh deleted file mode 100755 index 56678fd2d..000000000 --- a/tournament/scripts/demo/server-docker.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -ex - -GIT_ROOT_DIR=$(git rev-parse --show-toplevel) -cd $GIT_ROOT_DIR - -# NOTE: First, you may need to build the Docker image -tournament/scripts/docker/build-image.sh - -docker run \ - --add-host=host.docker.internal:host-gateway \ - --env-file $GIT_ROOT_DIR/tournament/scripts/docker/local-pg-credentials.env \ - -it \ - -p 8080:8080 \ - --rm swarm \ No newline at end of file diff --git a/tournament/scripts/demo/server-native.sh b/tournament/scripts/demo/server-native.sh index a29a58990..96ad0f0a7 100755 --- a/tournament/scripts/demo/server-native.sh +++ b/tournament/scripts/demo/server-native.sh @@ -6,8 +6,7 @@ cd $(git rev-parse --show-toplevel) GIT_HASH=$(git rev-parse HEAD) -stack build --fast swarm:swarm-host-tournament && \ - stack exec swarm-host-tournament -- \ +cabal run -j -O0 swarm:swarm-host-tournament -- \ --native-dev \ --port 8080 \ --version $GIT_HASH \ diff --git a/tournament/scripts/deploy/redeploy-binary.sh b/tournament/scripts/deploy/redeploy-binary.sh new file mode 100755 index 000000000..d47aa723d --- /dev/null +++ b/tournament/scripts/deploy/redeploy-binary.sh @@ -0,0 +1,17 @@ +#!/bin/bash -ex + +cd $(git rev-parse --show-toplevel) + +tournament/scripts/docker/build-static-binary.sh + +INTERNAL_BINARY_NAME=tournament-bin +scp -r $INTERNAL_BINARY_NAME lightsail: + +rm $INTERNAL_BINARY_NAME + +CURRENT_GIT_HASH_FILEPATH=git-hash.txt +git rev-parse HEAD > $CURRENT_GIT_HASH_FILEPATH +scp $CURRENT_GIT_HASH_FILEPATH lightsail: +rm $CURRENT_GIT_HASH_FILEPATH + +ssh lightsail -C 'sudo systemctl restart swarm-tournament' diff --git a/tournament/scripts/deploy/redeploy-web-files.sh b/tournament/scripts/deploy/redeploy-web-files.sh new file mode 100755 index 000000000..87a179b29 --- /dev/null +++ b/tournament/scripts/deploy/redeploy-web-files.sh @@ -0,0 +1,6 @@ +#!/bin/bash -ex + +cd $(git rev-parse --show-toplevel) + +scp -r tournament/web lightsail:tournament +scp -r data lightsail:.local/share/swarm diff --git a/tournament/scripts/docker/Dockerfile b/tournament/scripts/docker/Dockerfile deleted file mode 100644 index f685bc83e..000000000 --- a/tournament/scripts/docker/Dockerfile +++ /dev/null @@ -1,101 +0,0 @@ -# This is meant to be invoked while -# the CWD is the swarm repository root. - -FROM amazonlinux:latest as amz -LABEL org.opencontainers.image.authors="Karl Ostmo " - -ENV TZ=America/Los_Angeles - -# OS dependencies -RUN yum -y update && yum -y install \ - postgresql-devel - -# The 'python' executable is required to run the AWS CLI installer -#RUN yum -y install python -RUN ln -s /usr/bin/python3 /usr/bin/python - -RUN curl https://s3.amazonaws.com/aws-cli/awscli-bundle.zip -o awscli-bundle.zip -RUN unzip awscli-bundle.zip -RUN ./awscli-bundle/install -i /usr/local/aws -b /usr/local/bin/aws - -RUN mkdir -p /opt/swarm - -FROM amz as system-build-deps - -# These packages are only needed at build time, not at runtime. -RUN yum -y install \ - ncurses-compat-libs \ - gmp \ - gmp-devel \ - zlib-devel \ - fftw3-devel \ - xz-devel \ - nss \ - nss-devel \ - openssl-devel \ - gcc \ - gcc-c++ \ - make \ - tar - -FROM system-build-deps as haskell-compilation-layer - -# install ghcup -RUN \ - curl https://downloads.haskell.org/~ghcup/x86_64-linux-ghcup > /usr/bin/ghcup && \ - chmod +x /usr/bin/ghcup - -ARG GHC=9.6.4 - -# install GHC and cabal -RUN \ - ghcup -v install ghc --isolate /usr/local --force ${GHC} && \ - ghcup -v install cabal --isolate /usr/local/bin - - -WORKDIR /opt/swarm - -COPY ./swarm.cabal /opt/swarm/swarm.cabal -RUN cabal update - -# Must manually list the transitive closure of "internal" dependencies (sublibraries) -# of our executable. -# Note that we avoid simply listing all sublibraries here -# (i.e. scripts/gen/list-sublibraries.sh) because that -# includes 'swarm:swarm-web' and will build pandoc. -RUN cabal build --only-dependencies \ - swarm:swarm-host-tournament \ - swarm:swarm-tournament \ - swarm:swarm-engine \ - swarm:swarm-lang \ - swarm:swarm-scenario \ - swarm:swarm-util - -COPY ./src /opt/swarm/src -COPY ./app /opt/swarm/app - -# The following are not strictly needed for compiling the -# selected dependencies, but 'cabal build' spews warnings -# when they are absent -COPY ./test /opt/swarm/test -COPY ./CHANGELOG.md /opt/swarm/CHANGELOG.md -COPY ./LICENSE /opt/swarm/LICENSE - -COPY tournament/scripts/docker/build-server-executable.sh /opt/swarm/build-server-executable.sh -RUN /opt/swarm/build-server-executable.sh /opt/swarm/tournament-bin - -FROM amz - -COPY --from=haskell-compilation-layer /opt/swarm/tournament-bin /opt/swarm/tournament-bin -COPY ./data /root/.local/share/swarm/data -COPY ./tournament/web /root/tournament/web - -# This was produced by the parent script, 'build-image.sh'. -COPY ./git-hash.txt /root/git-hash.txt - -EXPOSE 8080 - -# We begin initially with CWD as the filesystem root, "/". -# We first 'cd' into the home directory, which is "/root", so we -# have access to the static web files. -CMD cd && /opt/swarm/tournament-bin --port 8080 --version $(cat git-hash.txt) diff --git a/tournament/scripts/docker/alpine/Dockerfile b/tournament/scripts/docker/alpine/Dockerfile new file mode 100644 index 000000000..17c800ffd --- /dev/null +++ b/tournament/scripts/docker/alpine/Dockerfile @@ -0,0 +1,57 @@ +# This is meant to be invoked while +# the CWD is the swarm repository root. + +FROM quay.io/benz0li/ghc-musl:9.6.4 as hs + +LABEL org.opencontainers.image.authors="Karl Ostmo " +ENV TZ=America/Los_Angeles + +RUN \ + apk add --no-cache git curl gcc g++ gmp-dev ncurses-dev libffi-dev make xz tar perl && \ + apk add --no-cache zlib zlib-dev zlib-static ncurses-static + +# install ghcup +RUN \ + curl https://downloads.haskell.org/~ghcup/x86_64-linux-ghcup > /usr/bin/ghcup && \ + chmod +x /usr/bin/ghcup + +ARG GHC=9.6.4 + +# install GHC and cabal +RUN ghcup -v install ghc --isolate /usr/local --force ${GHC} + +RUN mkdir -p /opt/swarm + +FROM hs as system-build-deps + +WORKDIR /opt/swarm + +COPY ./swarm.cabal /opt/swarm/swarm.cabal +RUN cabal update + +# Must manually list the transitive closure of "internal" dependencies (sublibraries) +# of our executable. +# Note that we avoid simply listing all sublibraries here +# (i.e. scripts/gen/list-sublibraries.sh) because that +# includes 'swarm:swarm-web' and will build pandoc. +RUN cabal build --only-dependencies \ + swarm:swarm-host-tournament \ + swarm:swarm-tournament \ + swarm:swarm-engine \ + swarm:swarm-lang \ + swarm:swarm-scenario \ + swarm:swarm-util + +COPY ./src /opt/swarm/src +COPY ./app /opt/swarm/app + +# The following are not strictly needed for compiling the +# selected dependencies, but 'cabal build' spews warnings +# when they are absent +COPY ./test /opt/swarm/test +COPY ./CHANGELOG.md /opt/swarm/CHANGELOG.md +COPY ./LICENSE /opt/swarm/LICENSE + +COPY ./tournament/scripts/docker/build-server-executable.sh /opt/swarm/build-server-executable.sh + +RUN cd /opt/swarm && ./build-server-executable.sh /opt/swarm/tournament-bin diff --git a/tournament/scripts/docker/aws-login.sh b/tournament/scripts/docker/aws-login.sh deleted file mode 100755 index bc03640b0..000000000 --- a/tournament/scripts/docker/aws-login.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -ex - -cd $(git rev-parse --show-toplevel) - -aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 254464607561.dkr.ecr.us-east-1.amazonaws.com/swarm-game \ No newline at end of file diff --git a/tournament/scripts/docker/build-image.sh b/tournament/scripts/docker/build-image.sh deleted file mode 100755 index 35314bbde..000000000 --- a/tournament/scripts/docker/build-image.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -ex - -cd $(git rev-parse --show-toplevel) - -CURRENT_GIT_HASH_FILEPATH=git-hash.txt - -git rev-parse HEAD > $CURRENT_GIT_HASH_FILEPATH - -docker build --tag swarm --file tournament/scripts/docker/Dockerfile . - -rm $CURRENT_GIT_HASH_FILEPATH \ No newline at end of file diff --git a/tournament/scripts/docker/build-server-executable.sh b/tournament/scripts/docker/build-server-executable.sh index 41858e80f..b36fe14f0 100755 --- a/tournament/scripts/docker/build-server-executable.sh +++ b/tournament/scripts/docker/build-server-executable.sh @@ -1,4 +1,4 @@ -#!/bin/bash -ex +#!/bin/sh -ex # Usage: # Intended to be invoked in Dockerfile. @@ -8,9 +8,8 @@ # Note that we use 'cabal' instead of 'stack' becuase # 'stack' fails to compile the 'vty' package within the Amazon Linux docker image. -# For faster development iteration, disable optimizations: -CABAL_ARGS="--disable-optimization swarm:swarm-host-tournament" -#CABAL_ARGS="swarm:swarm-host-tournament" +BUILD_TARGET=swarm:swarm-host-tournament +CABAL_ARGS="-j -O0 --enable-executable-static $BUILD_TARGET" -cabal build -j $CABAL_ARGS -cp $(cabal list-bin $CABAL_ARGS) $1 \ No newline at end of file +cabal build $CABAL_ARGS +cp $(cabal list-bin $CABAL_ARGS) $1 diff --git a/tournament/scripts/docker/build-static-binary.sh b/tournament/scripts/docker/build-static-binary.sh new file mode 100755 index 000000000..c0cef1d80 --- /dev/null +++ b/tournament/scripts/docker/build-static-binary.sh @@ -0,0 +1,14 @@ +#!/bin/bash -ex + +cd $(git rev-parse --show-toplevel) + +TAG_NAME=swarm +docker build --tag $TAG_NAME --file tournament/scripts/docker/alpine/Dockerfile . + +INTERNAL_BINARY_NAME=tournament-bin + +ID=$(docker create $TAG_NAME) +docker cp $ID:/opt/swarm/$INTERNAL_BINARY_NAME - | tar xv +docker rm -v $ID + +strip $INTERNAL_BINARY_NAME \ No newline at end of file diff --git a/tournament/scripts/docker/docker-prereqs.sh b/tournament/scripts/docker/docker-prereqs.sh deleted file mode 100644 index a726b6a75..000000000 --- a/tournament/scripts/docker/docker-prereqs.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -# Run this script on your host machine to prepare -# for development with Docker - -sudo apt install docker.io diff --git a/tournament/scripts/docker/local-docker-shell.sh b/tournament/scripts/docker/local-docker-shell.sh deleted file mode 100755 index 6d0123046..000000000 --- a/tournament/scripts/docker/local-docker-shell.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -ex - -cd $(git rev-parse --show-toplevel) - -#$NAME_ARG="--name swarm-container" -docker run -it $NAME_ARG -p 8080:8080 --entrypoint /bin/bash swarm \ No newline at end of file diff --git a/tournament/scripts/docker/local-pg-credentials.env b/tournament/scripts/docker/local-pg-credentials.env deleted file mode 100644 index ceabd3fe6..000000000 --- a/tournament/scripts/docker/local-pg-credentials.env +++ /dev/null @@ -1 +0,0 @@ -LOCAL_PGPASS=irrelevantpassword \ No newline at end of file diff --git a/tournament/scripts/docker/redeploy-image.sh b/tournament/scripts/docker/redeploy-image.sh deleted file mode 100755 index a7cb5103f..000000000 --- a/tournament/scripts/docker/redeploy-image.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -ex - -cd $(git rev-parse --show-toplevel) - -tournament/scripts/docker/build-image.sh - -AWS_DOCKER_IMAGE=254464607561.dkr.ecr.us-east-1.amazonaws.com/swarm-game:latest - -docker tag swarm:latest $AWS_DOCKER_IMAGE - -# Optionally log in again -tournament/scripts/docker/aws-login.sh -docker push $AWS_DOCKER_IMAGE - -# Next, run: -# eb deploy swarm-tournament-server-env \ No newline at end of file