feat(s3): add S3Storage adapter

libs/utils/src/deps.deno.ts

@@ -1,2 +1,2 @@
 export { Bot, Context } from 'https://lib.deno.dev/x/[email protected]/mod.ts';
-export type { SessionFlavor } from 'https://lib.deno.dev/x/[email protected]/mod.ts';
+export type { SessionFlavor, LazySessionFlavor } from 'https://lib.deno.dev/x/[email protected]/mod.ts';

libs/utils/src/deps.node.ts

@@ -1,2 +1,2 @@
 export { Bot, Context } from 'grammy';
-export type { SessionFlavor } from 'grammy';
+export type { SessionFlavor, LazySessionFlavor } from 'grammy';

libs/utils/src/mod.ts

@@ -10,11 +10,13 @@ interface StringSessionFlavor {
 }
 
 type JsonBot = Deps.Context & Deps.SessionFlavor<JsonSessionData>
+type LazyJsonBot = Deps.Context & Deps.LazySessionFlavor<JsonSessionData>
 type StringBot = Deps.Context & StringSessionFlavor
 
-export function createBot(json?: true): Deps.Bot<JsonBot>
-export function createBot(json?: false): Deps.Bot<StringBot>
-export function createBot(json = true) {
+export function createBot(json: true): Deps.Bot<JsonBot>
+export function createBot(json: true, lazy: true): Deps.Bot<LazyJsonBot>
+export function createBot(json: false): Deps.Bot<StringBot>
+export function createBot(json = true, lazy:boolean = false): Deps.Bot<any> {
   const botInfo = {
     id: 42,
     first_name: 'Test Bot',
@@ -26,7 +28,7 @@ export function createBot(json = true) {
   };
 
   if (json) {
-    return new Deps.Bot<JsonBot>('fake-token', { botInfo });
+    return new Deps.Bot<typeof lazy extends true ? LazyJsonBot : JsonBot>('fake-token', { botInfo });
   } else {
     return new Deps.Bot<StringBot>('fake-token', { botInfo });
   }
@@ -35,12 +37,12 @@ export function createBot(json = true) {
 export function createMessage(bot: Deps.Bot<any>, text = 'Test Text') {
   const createRandomNumber = () => Math.floor(Math.random() * (123456789 - 1) + 1);
 
-  const ctx = new Deps.Context({ 
-    update_id: createRandomNumber(), 
-    message: { 
+  const ctx = new Deps.Context({
+    update_id: createRandomNumber(),
+    message: {
       text,
       message_id: createRandomNumber(),
-      chat: { 
+      chat: {
         id: 1,
         type: 'private',
         first_name: 'Test User',
@@ -53,9 +55,9 @@ export function createMessage(bot: Deps.Bot<any>, text = 'Test Text') {
       },
     },
   },
-  bot.api, 
+  bot.api,
   bot.botInfo
   );
 
   return ctx;
-}
+}

packages/s3/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022-2024 Satont
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

packages/s3/README.md

@@ -0,0 +1,113 @@
+# S3 Storage Adapter for grammY
+
+Session storage adapter that can be used to
+[store your session data](https://grammy.dev/plugins/session.html) via an
+[S3 compatible object storage](https://en.wikipedia.org/wiki/Amazon_S3#S3_API_and_competing_services).
+
+The most prominent options are:
+
+- AWS S3 (12 months limited free tier)
+- Cloudflare R2 (unlimited free tier, no egress fee, but needs account with
+  payment connected, in case the use exceeds free tier)
+- https://github.com/minio/minio your own S3 in docker
+- ... <!-- is there a stable external list that compares the options? -->
+
+## Pros and Cons
+
+The biggest restriction of the current setup is that it only works with deno.
+For it to work with pnpm / node,
+we would need to add the following line to `.npmrc` (which is currently `.gitignore`d)
+```
+@jsr:registry=https://npm.jsr.io
+# The above doesn't work and prevents us from adding
+# "@bradenmacdonald/s3-lite-client": "npm:@jsr/[email protected]"
+# to packages/s3/package.json
+# the error reported by "pnpm i":
+#   ERR_PNPM_FETCH_404 GET https://registry.npmjs.org/@grammyjs%2Fstorage-utils: Not Found - 404
+#   This error happened while installing a direct dependency of /run/media/karfau/hdd-data/dev/storages/packages/file
+#   @grammyjs/storage-utils is not in the npm registry, or you have no permission to fetch it.
+```
+which would allow us to add `packages/s3/package.json` with the following `dependency`:
+`"@bradenmacdonald/s3-lite-client": "npm:@jsr/[email protected]"`
+it would also require some changes to the imports, to provide the bare specifiers as listed in `package.json`
+in `deno.json` "imports".
+And we would need to add related tests
+
+1. It is not the fastest way to get your data (benchmarks?), so it currently
+   does not implement the methods for loading all sessions. In a webhook
+   approach it works best using `LazySession`
+   [and the `serialize` middleware from `@grammyjs/runner`](https://grammy.dev/advanced/deployment#webhooks).
+2. You should consider limiting the key to
+   ["safe characters"](https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html).
+3. The setup requires 3-5 parameters (depending on the provider) that you need
+   to pass as env variables, which could be passed as one JSON string in a
+   single env variable. But by using individual andcommon variable names like
+   `AWS_SECRET_KEY`, more tools will pick them up. The process of getting all
+   the right parameters from your provider can be very different.
+4. Each provider can have different limitations regarding the storage, check
+   them out.
+5. You can use the same storage to even store the raw Updates to process them
+   later
+6. You can use the same storage to also store and access other data using the
+   `client`. It helps to think of the objects that you store as files on disk,
+   where you have at least one ID as a path element (or filename). Examples:
+   assets, or each update as a json file for async processing.
+7. You can use the aws cli, `mc`, `rclone` and similar CLI tools to access the
+   data or to get a local copy of some or all files.
+
+## Instructions
+
+1. Import the adapter
+
+   ```ts
+   import { S3DBAdapter } from "https://deno.land/x/grammy_storages/s3/src/mod.ts";
+   ```
+
+2. Get the credentials from you provider and pass them from one or multiple env
+   variables.
+
+   ```ts
+   const clientOptions: S3ClientOptions = JSON.parse(
+     Deno.env.get("S3_CLIENT_OPTS") ?? "{}",
+   );
+   ```
+
+3. Define lazy session structure
+
+   ```ts
+   interface SessionData {
+     count: number;
+   }
+   type MyContext = Context & LazySessionFlavor<SessionData>;
+   ```
+
+4. Define method to create session key from context
+
+   ```ts
+   const getSessionKey = (ctx: MyContext) =>
+     // it could be user based
+     `/chat/${ctx.from?.id ?? 0}/sesssion.json`;
+   // or if group chats are relevant, it could be chat based
+   //`/chat/${ctx.chat?.id ?? 0}/sesssion.json`;
+   ```
+
+5. Register adapter's middleware
+
+   ```ts
+   const bot = new Bot<MyContext>("<Token>");
+
+   bot.use(sequentialize(getSessionKey)).use(lazySession({
+     getSessionKey,
+     initial: () => ({ count: 0 }),
+     storage: new S3Adapter(clientOptions),
+   }));
+   ```
+
+Use `await ctx.session` as explained in
+[session plugin](https://grammy.dev/plugins/session.html#lazy-sessions)'s docs.
+
+<!--
+## More examples
+
+can be found in the [examples](./examples) folder.
+-->

packages/s3/adapter.ts

@@ -0,0 +1,72 @@
+import { StorageAdapter } from 'https://lib.deno.dev/x/[email protected]/mod.ts';
+import { S3Client } from 'jsr:@bradenmacdonald/[email protected]';
+export { S3Client } from 'jsr:@bradenmacdonald/[email protected]';
+/**
+ * The type of the constructor argument for S3Client
+ */
+// vendored from https://github.com/bradenmacdonald/s3-lite-client/blob/b34c604d0e9c3741919c92bc0151ec7d13eae467/client.ts#L15C1-L29C2
+// can be removed when https://github.com/bradenmacdonald/s3-lite-client/pull/38 is merged
+export interface S3ClientOptions {
+  /** Hostname of the endpoint. Not a URL, just the hostname with no protocol or port. */
+  endPoint: string;
+  accessKey?: string;
+  secretKey?: string;
+  sessionToken?: string;
+  useSSL?: boolean | undefined;
+  port?: number | undefined;
+  /** Default bucket name, if not specified on individual requests */
+  bucket?: string;
+  /** Region to use, e.g. "us-east-1" */
+  region: string;
+  /** Use path-style requests, e.g. https://endpoint/bucket/object-key instead of https://bucket/object-key (default: true) */
+  pathStyle?: boolean | undefined;
+}
+
+export type S3StorageClient = Pick<
+S3Client,
+'exists' | 'deleteObject' | 'getObject' | 'host' | 'region' | 'putObject'
+>;
+const isS3StorageClient = (
+  maybeClient: S3StorageClient | any,
+): maybeClient is S3StorageClient =>
+  ['exists', 'deleteObject', 'getObject', 'putObject'].every((required) =>
+    typeof maybeClient[required] === 'function'
+  );
+
+export const isObjectSession = (maybeSession: any): maybeSession is object =>
+  !!maybeSession && typeof maybeSession === 'object';
+
+export class S3Storage<T> implements StorageAdapter<T> {
+  readonly client: S3StorageClient;
+  constructor(
+    clientOrOptions: S3StorageClient | S3ClientOptions,
+    readonly validateSession: (data: any) => boolean,
+  ) {
+    this.client = isS3StorageClient(clientOrOptions)
+      ? clientOrOptions
+      : new S3Client(clientOrOptions);
+  }
+  /**
+   * @see https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html
+   */
+  readonly isSafe = (key: string): boolean => /^[-0-9a-zA-Z!_.()]+$/.test(key);
+
+  readonly delete = (key: string): Promise<void> =>
+    this.client.deleteObject(key);
+  readonly has = (key: string): Promise<boolean> => this.client.exists(key);
+  readonly read = async (key: string): Promise<T | undefined> => {
+    try {
+      const res = await this.client.getObject(key);
+      const data = await res.json() as T;
+      return this.validateSession(data) ? data : undefined;
+    } catch {
+      return undefined;
+    }
+  };
+
+  readonly write = async (key: string, value: T): Promise<void> => {
+    // the client has a mismatching return type
+    // to make type checks happy we await it, and intentionally do not return it
+    await this.client.putObject(key, JSON.stringify(value));
+  };
+}

packages/s3/deno.json

@@ -0,0 +1,8 @@
+{
+  "tasks": {
+    "check": "deno check *.ts test/*.ts",
+    "test": "deno test test"
+  },
+  "lock": false,
+  "nodeModulesDir": false
+}

packages/s3/package.json

@@ -0,0 +1,7 @@
+{
+  "name": "s3",
+  "private": true,
+  "scripts": {
+    "test:deno": "deno task test"
+  }
+}