feat(s3): add S3Storage adapter

.gitignore

@@ -31,10 +31,10 @@ yarn-error.log*
 .pnpm-debug.log
 dist
 eggs-debug.log
-.npmrc
+
 .pnpm-store
 .idea
 lerna-debug.log
 # pnpm-lock.yaml
 .env
-.wrangler
+.wrangler

.npmrc

@@ -0,0 +1 @@
+@jsr:registry=https://npm.jsr.io

libs/utils/src/deps.deno.ts

@@ -1,2 +1,2 @@
 export { Bot, Context } from 'https://lib.deno.dev/x/[email protected]/mod.ts';
-export type { SessionFlavor } from 'https://lib.deno.dev/x/[email protected]/mod.ts';
+export type { SessionFlavor, LazySessionFlavor } from 'https://lib.deno.dev/x/[email protected]/mod.ts';

libs/utils/src/deps.node.ts

@@ -1,2 +1,2 @@
 export { Bot, Context } from 'grammy';
-export type { SessionFlavor } from 'grammy';
+export type { SessionFlavor, LazySessionFlavor } from 'grammy';

libs/utils/src/mod.ts

@@ -10,11 +10,13 @@ interface StringSessionFlavor {
 }
 
 type JsonBot = Deps.Context & Deps.SessionFlavor<JsonSessionData>
+type LazyJsonBot = Deps.Context & Deps.LazySessionFlavor<JsonSessionData>
 type StringBot = Deps.Context & StringSessionFlavor
 
-export function createBot(json?: true): Deps.Bot<JsonBot>
-export function createBot(json?: false): Deps.Bot<StringBot>
-export function createBot(json = true) {
+export function createBot(json: true): Deps.Bot<JsonBot>
+export function createBot(json: true, lazy: true): Deps.Bot<LazyJsonBot>
+export function createBot(json: false): Deps.Bot<StringBot>
+export function createBot(json = true, lazy:boolean = false): Deps.Bot<any> {
   const botInfo = {
     id: 42,
     first_name: 'Test Bot',
@@ -26,7 +28,7 @@ export function createBot(json = true) {
   };
 
   if (json) {
-    return new Deps.Bot<JsonBot>('fake-token', { botInfo });
+    return new Deps.Bot<typeof lazy extends true ? LazyJsonBot : JsonBot>('fake-token', { botInfo });
   } else {
     return new Deps.Bot<StringBot>('fake-token', { botInfo });
   }
@@ -35,12 +37,12 @@ export function createBot(json = true) {
 export function createMessage(bot: Deps.Bot<any>, text = 'Test Text') {
   const createRandomNumber = () => Math.floor(Math.random() * (123456789 - 1) + 1);
 
-  const ctx = new Deps.Context({ 
-    update_id: createRandomNumber(), 
-    message: { 
+  const ctx = new Deps.Context({
+    update_id: createRandomNumber(),
+    message: {
       text,
       message_id: createRandomNumber(),
-      chat: { 
+      chat: {
         id: 1,
         type: 'private',
         first_name: 'Test User',
@@ -53,9 +55,9 @@ export function createMessage(bot: Deps.Bot<any>, text = 'Test Text') {
       },
     },
   },
-  bot.api, 
+  bot.api,
   bot.botInfo
   );
 
   return ctx;
-}
+}

package.json

@@ -31,5 +31,6 @@
   "engines": {
     "pnpm": ">=7.0.0",
     "node": ">=12.0.0"
-  }
+  },
+  "packageManager": "[email protected]+sha1.648f6014eb363abb36618f2ba59282a9eeb3e879"
 }

packages/README.md

@@ -31,7 +31,7 @@ Bug reports and pull requests are welcome.
   │       │             │
   │       │             └─⫸ Summary in present tense. Not capitalized. No period at the end.
   │       │
-  │       └─⫸ Commit Scope: utils|file|mongodb|psql|redis|typeorm|supabase|free|firestore|deta|denodb|denokv|cloudflare
+  │       └─⫸ Commit Scope: cloudflare|denodb|denokv|deta|file|firestore|free|mongodb|psql|redis|s3|supabase|typeorm|utils
   │                          
   │                          
   │                          

packages/file/package.json

@@ -36,7 +36,7 @@
   },
   "homepage": "https://github.com/grammyjs/storages/tree/main/packages/file#readme",
   "devDependencies": {
-    "@grammyjs/storage-utils": "^2.4.2",
+    "@grammyjs/storage-utils": "workspace:*",
     "grammy": "^1.21.1"
   },
   "gitHead": "a7758c4f957f103a14832088c6858d693c444576"

packages/mongodb/package.json

@@ -36,7 +36,7 @@
   },
   "homepage": "https://github.com/grammyjs/storages/tree/main/packages/mongodb#readme",
   "devDependencies": {
-    "@grammyjs/storage-utils": "^2.4.2",
+    "@grammyjs/storage-utils": "workspace:*",
     "grammy": "^1.21.1",
     "mongodb": "^6.3.0",
     "mongodb-memory-server": "^9.1.6"

packages/prisma/package.json

@@ -34,7 +34,7 @@
   },
   "homepage": "https://github.com/grammyjs/storages/tree/main/packages/typeorm#readme",
   "devDependencies": {
-    "@grammyjs/storage-utils": "^2.4.2",
+    "@grammyjs/storage-utils": "workspace:*",
     "@prisma/client": "^5.10.2",
     "grammy": "^1.21.1",
     "prisma": "^5.10.2",

packages/psql/package.json

@@ -36,7 +36,7 @@
   },
   "homepage": "https://github.com/grammyjs/storages/tree/main/packages/psql#readme",
   "devDependencies": {
-    "@grammyjs/storage-utils": "^2.4.2",
+    "@grammyjs/storage-utils": "workspace:*",
     "@types/pg": "^8.11.2",
     "grammy": "^1.21.1",
     "pg": "^8.11.3",

packages/redis/package.json

@@ -36,7 +36,7 @@
   },
   "homepage": "https://github.com/grammyjs/storages/tree/main/packages/redis#readme",
   "devDependencies": {
-    "@grammyjs/storage-utils": "^2.4.2",
+    "@grammyjs/storage-utils": "workspace:*",
     "@types/ioredis": "^5.0.0",
     "grammy": "^1.21.1",
     "ioredis": "^5.3.2"

packages/s3/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Christian Bewernitz
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

packages/s3/README.md

@@ -0,0 +1,121 @@
+# S3 Storage Adapter for grammY
+
+Session storage adapter that can be used to
+[store your session data](https://grammy.dev/plugins/session.html) via an
+[S3 compatible object storage](https://en.wikipedia.org/wiki/Amazon_S3#S3_API_and_competing_services).
+
+The most prominent options are:
+
+- AWS S3 (12 months limited free tier)
+- Cloudflare R2 (unlimited free tier, no egress fee, but needs account with
+  payment connected, in case the use exceeds free tier)
+- https://github.com/minio/minio your own S3 in docker
+- ... <!-- is there a stable external list that compares the options? -->
+
+## Pros and Cons
+
+The biggest restriction of the current setup is that it only works with deno.
+For it to work with pnpm / node, we would need to add the following line to
+`.npmrc` (which is currently `.gitignore`d)
+
+```
+@jsr:registry=https://npm.jsr.io
+# The above doesn't work and prevents us from adding
+# "@bradenmacdonald/s3-lite-client": "npm:@jsr/[email protected]"
+# to packages/s3/package.json
+# the error reported by "pnpm i":
+#   ERR_PNPM_FETCH_404 GET https://registry.npmjs.org/@grammyjs%2Fstorage-utils: Not Found - 404
+#   This error happened while installing a direct dependency of /run/media/karfau/hdd-data/dev/storages/packages/file
+#   @grammyjs/storage-utils is not in the npm registry, or you have no permission to fetch it.
+```
+
+which would allow us to add `packages/s3/package.json` with the following
+`dependency`:
+`"@bradenmacdonald/s3-lite-client": "npm:@jsr/[email protected]"`
+it would also require some changes to the imports, to provide the bare
+specifiers as listed in `package.json` in `deno.json` "imports". And we would
+need to add related tests
+
+1. It is not the fastest way to get your data (benchmarks?), so it currently
+   does not implement the methods for loading all sessions. In a webhook
+   approach it works best using `LazySession`
+   [and the `serialize` middleware from `@grammyjs/runner`](https://grammy.dev/advanced/deployment#webhooks).
+2. You should consider limiting the key to
+   ["safe characters"](https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html).
+3. The setup requires 3-5 parameters (depending on the provider) that you need
+   to pass as env variables, which could be passed as one JSON string in a
+   single env variable. But by using individual andcommon variable names like
+   `AWS_SECRET_KEY`, more tools will pick them up. The process of getting all
+   the right parameters from your provider can be very different.
+4. Each provider can have different limitations regarding the storage, check
+   them out.
+5. You can use the same storage to even store the raw Updates to process them
+   later
+6. You can use the same storage to also store and access other data using the
+   `client`. It helps to think of the objects that you store as files on disk,
+   where you have at least one ID as a path element (or filename). Examples:
+   assets, or each update as a json file for async processing.
+7. You can use the aws cli, `mc`, `rclone` and similar CLI tools to access the
+   data or to get a local copy of some or all files.
+
+## Instructions
+
+1. Import the adapter
+
+   ```ts
+   import { S3DBAdapter } from "https://deno.land/x/grammy_storages/s3/src/mod.ts";
+   ```
+
+2. Get the credentials from you provider and pass them from one or multiple env
+   variables.
+
+   ```ts
+   const clientOptions: S3ClientOptions = JSON.parse(
+     Deno.env.get("S3_CLIENT_OPTS") ?? "{}",
+   );
+   ```
+
+3. Define lazy session structure
+
+   ```ts
+   interface SessionData {
+     count: number;
+   }
+   type MyContext = Context & LazySessionFlavor<SessionData>;
+   ```
+
+4. Define method to create session key from context
+
+   ```ts
+   function getSessionKey(ctx: MyContext) {
+     // it could be user based
+     return `/chat/${ctx.from?.id ?? 0}/session.json`;
+     // or if group chats are relevant, it could be chat based
+     // return `/chat/${ctx.chat?.id ?? 0}/session.json`;
+   }
+   ```
+
+5. Register adapter's middleware
+
+   ```ts
+   const bot = new Bot<MyContext>("<Token>");
+
+   bot.use(sequentialize(getSessionKey)).use(
+     lazySession({
+       getSessionKey,
+       initial() {
+         return { count: 0 };
+       },
+       storage: new S3Adapter(clientOptions),
+     }),
+   );
+   ```
+
+Use `await ctx.session` as explained in
+[session plugin](https://grammy.dev/plugins/session.html#lazy-sessions)'s docs.
+
+<!--
+## More examples
+
+can be found in the [examples](./examples) folder.
+-->

packages/s3/adapter.ts

@@ -0,0 +1,66 @@
+import {
+  S3Client,
+  type S3ClientOptions,
+  type StorageAdapter,
+} from './deps.deno.ts';
+export { S3Client, type S3ClientOptions } from './deps.deno.ts';
+
+export type S3StorageClient = Pick<
+S3Client,
+'exists' | 'deleteObject' | 'getObject' | 'host' | 'region' | 'putObject'
+>;
+function isS3StorageClient(
+  maybeClient: S3StorageClient | S3ClientOptions,
+): maybeClient is S3StorageClient {
+  return ['exists', 'deleteObject', 'getObject', 'putObject'].every(
+    (required) =>
+      typeof maybeClient[required as keyof typeof maybeClient] === 'function',
+  );
+}
+
+export function isObjectSession(maybeSession: unknown): maybeSession is object {
+  return !!maybeSession && typeof maybeSession === 'object';
+}
+
+export class S3Storage<T> implements StorageAdapter<T> {
+  readonly client: S3StorageClient;
+  constructor(
+    clientOrOptions: S3StorageClient | S3ClientOptions,
+    readonly validateSession: (data: T | unknown) => boolean,
+  ) {
+    this.client = isS3StorageClient(clientOrOptions)
+      ? clientOrOptions
+      : new S3Client(clientOrOptions);
+  }
+
+  /**
+   * @see https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html
+   */
+  isSafe(key: string): boolean {
+    return /^[-0-9a-zA-Z!_.()]+$/.test(key);
+  }
+
+  async delete(key: string): Promise<void> {
+    return await this.client.deleteObject(key);
+  }
+
+  async has(key: string): Promise<boolean> {
+    return await this.client.exists(key);
+  }
+
+  async read(key: string): Promise<T | undefined> {
+    try {
+      const res = await this.client.getObject(key);
+      const data = (await res.json()) as T;
+      return this.validateSession(data) ? data : undefined;
+    } catch {
+      return undefined;
+    }
+  }
+
+  async write(key: string, value: T): Promise<void> {
+    // the client has a mismatching return type
+    // to make type checks happy we await it, and intentionally do not return it
+    await this.client.putObject(key, JSON.stringify(value));
+  }
+}

packages/s3/deno.json

@@ -0,0 +1,15 @@
+{
+  "tasks": {
+    "check": "deno check *.ts test/*.ts",
+    "test": "deno test test"
+  },
+  "lock": false,
+  "nodeModulesDir": false,
+  "imports": {
+    "@bradenmacdonald/s3-lite-client": "jsr:@bradenmacdonald/[email protected]",
+    "@grammyjs/storage-utils": "../../libs/utils/src/mod.ts",
+    "@std/assert": "jsr:@std/assert",
+    "@std/testing/mock": "jsr:@std/testing/mock",
+    "grammy": "https://lib.deno.dev/x/[email protected]/mod.ts"
+  }
+}

packages/s3/deps.deno.ts

@@ -0,0 +1,5 @@
+export type { StorageAdapter } from 'grammy';
+export {
+  S3Client,
+  type S3ClientOptions,
+} from '@bradenmacdonald/s3-lite-client';

packages/s3/deps.node.ts

@@ -0,0 +1,5 @@
+export type { StorageAdapter } from 'grammy';
+export {
+  S3Client,
+  type S3ClientOptions,
+} from '@bradenmacdonald/s3-lite-client';

packages/s3/package.json

@@ -0,0 +1,10 @@
+{
+  "name": "s3",
+  "private": true,
+  "scripts": {
+    "test:deno": "deno task test"
+  },
+  "dependencies": {
+    "@bradenmacdonald/s3-lite-client": "@jsr/[email protected]"
+  }
+}