Skip to content

Commit

Permalink
Merge pull request #6 from theapexlab/db-setup
Browse files Browse the repository at this point in the history
APEX-1587 - db setup
  • Loading branch information
BaDo2001 authored Oct 31, 2023
2 parents 746d981 + 7c1911d commit 13cc07e
Show file tree
Hide file tree
Showing 26 changed files with 1,575 additions and 92 deletions.
20 changes: 19 additions & 1 deletion .env
Original file line number Diff line number Diff line change
@@ -1,3 +1,11 @@
# Postgres settings, used for local development, could be anything as long as they are consistent
POSTGRES_USER=
POSTGRES_PASSWORD=
POSTGRES_DB=
DB_URL=
# Should be set to true if you want to use a local database instead of the deployed one (recommended for local development)
USE_LOCAL_DB=

# The ApiEndpoint of the deployed backend
VITE_API_URL=
# The slack bot token
Expand All @@ -11,4 +19,14 @@ VITE_SLACK_BOT_USER_ID=
# The slack channel id used for testing
VITE_CORE_SLACK_CHANNEL_ID=
# The slack dm id between the bot and you
VITE_SLACK_DM_ID=
VITE_SLACK_DM_ID=
# The url of the local database
VITE_DB_URL=
# Used to indicate that the tests are running in a ci environment (not needed for local development)
VITE_CI=
# The name of the deployed RDS database (not needed for local development)
VITE_DB_NAME=
# The secret arn of the deployed RDS database (not needed for local development)
VITE_DB_SECRET_ARN=
# The cluster arn of the deployed RDS database (not needed for local development)
VITE_DB_CLUSTER_ARN=
17 changes: 16 additions & 1 deletion .github/workflows/pr.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -76,11 +76,22 @@ jobs:
- name: Deploy stack
run: pnpm run deploy --stage ${{ env.STAGE }}

- name: Extract api endpoint
- name: Extract stack outputs
id: sst-output
run: |
URL=$(jq -r '.["${{env.STAGE}}-birthday-slack-bot-MyStack"].ApiEndpoint' .sst/outputs.json)
echo "apiEndpoint=$URL" >> "$GITHUB_OUTPUT"
MIGRATION_FUNCTION=$(jq -r '.["${{env.STAGE}}-birthday-slack-bot-MyStack"].MigrationFunctionName' .sst/outputs.json)
echo "migrationFunction=$MIGRATION_FUNCTION" >> "$GITHUB_OUTPUT"
SECRET_ARN=$(jq -r '.["${{env.STAGE}}-birthday-slack-bot-StorageStack"].RDSSECRETARN' .sst/outputs.json)
echo "secretArn=$SECRET_ARN" >> "$GITHUB_OUTPUT"
CLUSTER_ARN=$(jq -r '.["${{env.STAGE}}-birthday-slack-bot-StorageStack"].RDSCLUSTERARN' .sst/outputs.json)
echo "clusterArn=$CLUSTER_ARN" >> "$GITHUB_OUTPUT"
DATABASE=$(jq -r '.["${{env.STAGE}}-birthday-slack-bot-StorageStack"].RDSDATABASE' .sst/outputs.json)
echo "database=$DATABASE" >> "$GITHUB_OUTPUT"
- name: Migration
run: pnpm migration:ci ${{ steps.sst-output.outputs.migrationFunction }}

- name: Run integration tests
run: pnpm test:integration
Expand All @@ -92,3 +103,7 @@ jobs:
VITE_SLACK_BOT_USER_ID: ${{ secrets.SLACK_BOT_USER_ID_TEST }}
VITE_CORE_SLACK_CHANNEL_ID: ${{ secrets.CORE_SLACK_CHANNEL_ID_TEST }}
VITE_SLACK_DM_ID: ${{ secrets.SLACK_DM_ID_TEST }}
VITE_CI: true
VITE_DB_NAME: ${{ steps.sst-output.outputs.database }}
VITE_DB_SECRET_ARN: ${{ steps.sst-output.outputs.secretArn }}
VITE_DB_CLUSTER_ARN: ${{ steps.sst-output.outputs.clusterArn }}
9 changes: 9 additions & 0 deletions .github/workflows/staging.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -58,3 +58,12 @@ jobs:
- name: Deploy stack
run: pnpm run deploy --stage "${{ env.stage }}"

- name: Extract stack outputs
id: sst-output
run: |
MIGRATION_FUNCTION=$(jq -r '.["${{env.STAGE}}-birthday-slack-bot-MyStack"].MigrationFunctionName' .sst/outputs.json)
echo "migrationFunction=$MIGRATION_FUNCTION" >> "$GITHUB_OUTPUT"
- name: Migration
run: pnpm migration:ci ${{ steps.sst-output.outputs.migrationFunction }}
4 changes: 3 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -14,4 +14,6 @@ node_modules
# local env files
.env*.local

sib-report.txt
sib-report.txt

cdk.context.json
20 changes: 18 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,17 @@ npx sst secrets set RANDOM_SLACK_CHANNEL_ID <your-test-channel>
```

3. Install dependencies: `pnpm i`
4. Run sst: `pnpm dev`

## DB Setup

1. Make sure you have docker-compose available.
2. Copy the `.env` file to a `.env.local` file.
3. Run `docker-compose up -d` to start the database.
4. # Run `pnpm migration:local` to run the migrations.

## Run locally

`pnpm dev`

### Add webhook url to Slack

Expand All @@ -64,8 +74,14 @@ npx sst secrets set RANDOM_SLACK_CHANNEL_ID <your-test-channel>

## Run tests

Copy the `.env` file to a `.env.local` file and add the secrets.
Fill `.env.local` file with the secrets.

```bash
pnpm test
```

## Generate a new migration

```bash
pnpm migration:generate
```
7 changes: 7 additions & 0 deletions docker-compose.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
services:
postgres:
image: "postgres:13.9"
ports:
- "5432:5432"
env_file:
- .env.local
13 changes: 11 additions & 2 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
"private": true,
"type": "module",
"scripts": {
"dev": "sst dev",
"dev": "docker-compose up -d && sst dev",
"build": "sst build",
"deploy": "sst deploy",
"remove": "sst remove",
Expand All @@ -18,29 +18,38 @@
"prepare": "husky install",
"test": "vitest watch",
"test:unit": "vitest run unit",
"test:integration": "vitest run integration --threads false --single-thread"
"test:integration": "vitest run integration --threads false --single-thread",
"migration:generate": "cd packages/core && pnpm migration:generate",
"migration:local": "NODE_ENV=development tsx scripts/migrateLocal.ts",
"migration:ci": "tsx scripts/migrateCi.ts"
},
"dependencies": {
"sst": "^2.28.1"
},
"devDependencies": {
"@aws-sdk/client-eventbridge": "^3.429.0",
"@aws-sdk/client-lambda": "^3.438.0",
"@slack/bolt": "^3.14.0",
"@theapexlab/serverless-icebreaker": "^1.1.3",
"@tsconfig/node18": "^18.2.2",
"@types/aws-lambda": "^8.10.124",
"@types/pg": "^8.10.7",
"@typescript-eslint/eslint-plugin": "^6.7.5",
"@typescript-eslint/parser": "^6.7.5",
"aws-cdk-lib": "2.95.1",
"constructs": "10.2.69",
"dotenv": "^16.3.1",
"drizzle-orm": "^0.28.6",
"eslint": "^8.51.0",
"eslint-config-prettier": "^9.0.0",
"eslint-define-config": "^1.24.1",
"eslint-plugin-import": "^2.28.1",
"eslint-plugin-simple-import-sort": "^10.0.0",
"husky": "^8.0.0",
"lint-staged": "^14.0.1",
"pg": "^8.11.3",
"prettier": "^3.0.3",
"tsx": "^3.14.0",
"typescript": "^5.2.2",
"vite": "^4.5.0",
"vitest": "^0.34.6"
Expand Down
52 changes: 52 additions & 0 deletions packages/core/db/dbFactory.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
import { RDSDataClient } from "@aws-sdk/client-rds-data";
import { drizzle as drizzleRds } from "drizzle-orm/aws-data-api/pg";
import { migrate as migrateRds } from "drizzle-orm/aws-data-api/pg/migrator";
import { drizzle as drizzleNode } from "drizzle-orm/node-postgres";
import { migrate as migrateNode } from "drizzle-orm/node-postgres/migrator";
import pg from "pg";

const migrationsFolder = "./packages/core/db/migrations";

type FactoryPayload =
| {
type: "node";
connectionString: string;
}
| {
type: "aws";
database: string;
secretArn: string;
resourceArn: string;
};

export const dbFactory = (payload: FactoryPayload) => {
if (payload.type === "node") {
const pool = new pg.Pool({
...payload,
});

const db = drizzleNode(pool);

return [
db,
async () => {
await migrateNode(db, {
migrationsFolder,
});
},
] as const;
}

const db = drizzleRds(new RDSDataClient({}), {
...payload,
});

return [
db,
async () => {
await migrateRds(db, {
migrationsFolder,
});
},
] as const;
};
21 changes: 21 additions & 0 deletions packages/core/db/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
/* eslint-disable @typescript-eslint/ban-ts-comment */
import { RDS } from "sst/node/rds";

import { dbFactory } from "./dbFactory";

export const [db, migrate] = dbFactory(
process.env.IS_LOCAL || process.env.NODE_ENV === "development"
? {
type: "node",
connectionString: process.env.DB_URL ?? "",
}
: {
type: "aws",
//@ts-ignore
database: RDS.Database.defaultDatabaseName,
//@ts-ignore
secretArn: RDS.Database.secretArn,
//@ts-ignore
resourceArn: RDS.Database.clusterArn,
},
);
11 changes: 11 additions & 0 deletions packages/core/db/migrations/0000_clammy_vargas.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
CREATE TABLE IF NOT EXISTS "testItems" (
"id" varchar PRIMARY KEY NOT NULL,
"payload" varchar
);
--> statement-breakpoint
CREATE TABLE IF NOT EXISTS "users" (
"id" varchar,
"team_id" varchar,
"birthday" date,
CONSTRAINT users_id_team_id PRIMARY KEY("id","team_id")
);
73 changes: 73 additions & 0 deletions packages/core/db/migrations/meta/0000_snapshot.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
{
"version": "5",
"dialect": "pg",
"id": "43c150de-7698-45b1-9f52-8feb96883247",
"prevId": "00000000-0000-0000-0000-000000000000",
"tables": {
"testItems": {
"name": "testItems",
"schema": "",
"columns": {
"id": {
"name": "id",
"type": "varchar",
"primaryKey": true,
"notNull": true
},
"payload": {
"name": "payload",
"type": "varchar",
"primaryKey": false,
"notNull": false
}
},
"indexes": {},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {}
},
"users": {
"name": "users",
"schema": "",
"columns": {
"id": {
"name": "id",
"type": "varchar",
"primaryKey": false,
"notNull": false
},
"team_id": {
"name": "team_id",
"type": "varchar",
"primaryKey": false,
"notNull": false
},
"birthday": {
"name": "birthday",
"type": "date",
"primaryKey": false,
"notNull": false
}
},
"indexes": {},
"foreignKeys": {},
"compositePrimaryKeys": {
"users_id_team_id": {
"name": "users_id_team_id",
"columns": [
"id",
"team_id"
]
}
},
"uniqueConstraints": {}
}
},
"enums": {},
"schemas": {},
"_meta": {
"schemas": {},
"tables": {},
"columns": {}
}
}
13 changes: 13 additions & 0 deletions packages/core/db/migrations/meta/_journal.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
{
"version": "5",
"dialect": "pg",
"entries": [
{
"idx": 0,
"version": "5",
"when": 1698670168682,
"tag": "0000_clammy_vargas",
"breakpoints": true
}
]
}
18 changes: 18 additions & 0 deletions packages/core/db/schema.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
import { date, pgTable, primaryKey, varchar } from "drizzle-orm/pg-core";

export const users = pgTable(
"users",
{
id: varchar("id"),
teamId: varchar("team_id"),
birthday: date("birthday", { mode: "date" }),
},
(t) => ({
pk: primaryKey(t.id, t.teamId),
}),
);

export const testItems = pgTable("testItems", {
id: varchar("id").primaryKey(),
payload: varchar("payload"),
});
8 changes: 7 additions & 1 deletion packages/core/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,21 @@
"type": "module",
"scripts": {
"test": "sst bind vitest",
"typecheck": "tsc -noEmit"
"typecheck": "tsc -noEmit",
"migration:generate": "drizzle-kit generate:pg --schema=db/schema.ts --out=db/migrations"
},
"devDependencies": {
"@types/node": "^20.4.5",
"@types/pg": "^8.10.7",
"drizzle-kit": "^0.19.13",
"sst": "^2.22.9",
"vitest": "^0.33.0"
},
"dependencies": {
"@aws-sdk/client-rds-data": "^3.438.0",
"@slack/bolt": "^3.14.0",
"drizzle-orm": "^0.28.6",
"pg": "^8.11.3",
"zod": "^3.22.4"
}
}
Loading

0 comments on commit 13cc07e

Please sign in to comment.