feat(#441): Upload entities to s3 for faster api

packages/entity-postgres-s3-sink/.gitignore

@@ -0,0 +1,4 @@
+npm-debug.log
+dist/
+tmp/
+./node_modules

packages/entity-postgres-s3-sink/CHANGELOG.md

@@ -0,0 +1,80 @@
+# Change Log
+
+All notable changes to this project will be documented in this file.
+See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+
+# [0.4.0](https://github.com/helium/helium-program-libary/compare/v0.2.21...v0.4.0) (2023-09-22)
+
+**Note:** Version bump only for package @helium/account-postgres-sink-service
+
+
+
+
+
+## [0.3.2](https://github.com/helium/helium-program-libary/compare/v0.2.21...v0.3.2) (2023-09-16)
+
+**Note:** Version bump only for package @helium/account-postgres-sink-service
+
+
+
+
+
+## [0.3.1](https://github.com/helium/helium-program-libary/compare/v0.2.21...v0.3.1) (2023-09-15)
+
+**Note:** Version bump only for package @helium/account-postgres-sink-service
+
+
+
+
+
+# [0.3.0](https://github.com/helium/helium-program-libary/compare/v0.2.21...v0.3.0) (2023-09-14)
+
+**Note:** Version bump only for package @helium/account-postgres-sink-service
+
+
+
+
+
+## [0.2.22](https://github.com/helium/helium-program-libary/compare/v0.2.21...v0.2.22) (2023-09-13)
+
+**Note:** Version bump only for package @helium/account-postgres-sink-service
+
+
+
+
+
+## [0.2.15](https://github.com/helium/helium-program-libary/compare/v0.2.14...v0.2.15) (2023-07-31)
+
+**Note:** Version bump only for package @helium/account-postgres-sink-service
+
+
+
+
+
+## [0.2.3](https://github.com/helium/helium-program-libary/compare/v0.1.5...v0.2.3) (2023-06-16)
+
+**Note:** Version bump only for package @helium/account-postgres-sink-service
+
+
+
+
+
+## [0.2.2](https://github.com/helium/helium-program-libary/compare/v0.1.5...v0.2.2) (2023-06-08)
+
+**Note:** Version bump only for package @helium/account-postgres-sink-service
+
+
+
+
+
+## [0.2.1](https://github.com/helium/helium-program-libary/compare/v0.1.5...v0.2.1) (2023-06-08)
+
+**Note:** Version bump only for package @helium/account-postgres-sink-service
+
+
+
+
+
+# [0.2.0](https://github.com/helium/helium-program-libary/compare/v0.1.5...v0.2.0) (2023-06-06)
+
+**Note:** Version bump only for package @helium/account-postgres-sink-service

packages/entity-postgres-s3-sink/Dockerfile

@@ -0,0 +1,30 @@
+# Specify the base image
+FROM node:16-alpine AS BUILD_IMAGE
+
+WORKDIR /usr/src/app
+
+COPY package.json ./
+
+RUN yarn install
+
+COPY src src
+COPY tsconfig.build.json tsconfig.json
+
+RUN yarn global add typescript
+RUN yarn run build
+
+RUN npm prune --production
+
+FROM node:16-alpine
+
+WORKDIR /usr/src/app
+
+COPY --from=BUILD_IMAGE /usr/src/app/lib ./lib
+COPY --from=BUILD_IMAGE /usr/src/app/node_modules ./node_modules
+
+# This isn't actually used, service is read only. But anchor wants a wallet.
+RUN echo "[124,96,181,146,132,165,175,182,60,194,167,230,29,91,110,109,226,38,41,155,207,186,24,33,205,120,108,98,218,67,77,95,13,60,79,204,253,10,183,101,60,94,220,177,117,97,16,29,31,124,35,65,121,147,161,114,159,23,207,202,122,164,170,201]" > id.json
+
+ENV ANCHOR_WALLET=/usr/src/app/id.json
+
+CMD ["node", "lib/src/index.js"]

packages/entity-postgres-s3-sink/package.json

@@ -0,0 +1,68 @@
+{
+  "name": "@helium/entity-postgres-s3-sink",
+  "private": true,
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org/"
+  },
+  "license": "Apache-2.0",
+  "version": "0.4.0",
+  "description": "Sync account data to postgres",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/helium/helium-program-libary"
+  },
+  "main": "./lib/cjs/index.js",
+  "module": "./lib/esm/src/index.js",
+  "types": "./lib/types/src/index.d.ts",
+  "sideEffects": false,
+  "files": [
+    "lib"
+  ],
+  "exports": {
+    "import": "./lib/esm/src/index.js",
+    "require": "./lib/cjs/index.js",
+    "types": "./lib/types/src/index.d.ts"
+  },
+  "scripts": {
+    "format": "prettier --write \"src/**/*.{ts,tsx}\"",
+    "precommit": "npx git-format-staged -f 'prettier --ignore-unknown --stdin --stdin-filepath \"{}\"' .",
+    "build": "tsc -p tsconfig.json",
+    "start": "node lib/esm/server.js",
+    "dev": "npx ts-node --project tsconfig.cjs.json src/index.ts"
+  },
+  "dependencies": {
+    "@coral-xyz/anchor": "^0.28.0",
+    "@fastify/cors": "^8.1.1",
+    "@helium/address": "^4.10.2",
+    "@helium/account-fetch-cache": "^0.4.0",
+    "@metaplex-foundation/mpl-token-metadata": "^2.10.0",
+    "@solana/web3.js": "^1.78.4",
+    "aws-sdk": "^2.1344.0",
+    "axios": "^1.3.6",
+    "axios-retry": "^3.8.0",
+    "bn.js": "^5.2.0",
+    "bs58": "^4.0.1",
+    "h3-js": "^4.1.0",
+    "p-limit": "3.1.0",
+    "pg": "^8.9.0",
+    "prom-client": "^14.2.0",
+    "sequelize": "^6.28.0",
+    "yargs": "^17.7.1"
+  },
+  "devDependencies": {
+    "@types/bn.js": "^5.1.1",
+    "@types/deep-equal": "^1.0.1",
+    "@types/node": "^18.11.11",
+    "@types/pg": "^8.6.6",
+    "@types/yargs": "^17.0.24",
+    "git-format-staged": "^2.1.3",
+    "ts-loader": "^9.2.3",
+    "ts-node": "^10.9.1",
+    "ts-node-dev": "^2.0.0",
+    "typescript": "^4.8.4",
+    "yarn": "^1.22.18"
+  },
+  "keywords": [],
+  "author": ""
+}

packages/entity-postgres-s3-sink/program_account_configs_example.json

@@ -0,0 +1,121 @@
+{
+  "configs": [
+    {
+      "programId": "1atrmQs3eq1N2FEYWu6tyTXbCjP4uQwExpjtnhXtS8h",
+      "accounts": [
+        {
+          "type": "LazyTransactionsV0",
+          "table": "lazy_transactions",
+          "schema": "public"
+        },
+        { "type": "Block", "table": "blocks", "schema": "public" }
+      ]
+    },
+    {
+      "programId": "hvsrNC3NKbcryqDs2DocYHZ9yPKEVzdSjQG6RVtK1s8",
+      "accounts": [
+        { "type": "PositionV0", "table": "positions", "schema": "public" },
+        { "type": "Registrar", "table": "registrars", "schema": "public" }
+      ],
+      "crons": [
+        { "schedule": "*/15 * * * *", "type": "refresh-accounts" },
+        { "schedule": "*/15 * * * *", "type": "integrity-check" }
+      ]
+    },
+    {
+      "programId": "hdaoVTCqhfHHo75XdAMxBKdUqvq1i5bF23sisBqVgGR",
+      "accounts": [
+        { "type": "DaoV0", "table": "daos", "schema": "public" },
+        { "type": "SubDaoV0", "table": "sub_daos", "schema": "public" },
+        {
+          "type": "SubDaoEpochInfoV0",
+          "table": "sub_dao_epoch_infos",
+          "schema": "public"
+        },
+        {
+          "type": "DaoEpochInfoV0",
+          "table": "dao_epoch_infos",
+          "schema": "public"
+        },
+        {
+          "type": "DelegatedPositionV0",
+          "table": "delegated_positions",
+          "schema": "public"
+        }
+      ]
+    },
+    {
+      "programId": "circAbx64bbsscPbQzZAUvuXpHqrCe6fLMzc2uKXz9g",
+      "accounts": [
+        {
+          "type": "MintWindowedCircuitBreakerV0",
+          "table": "mint_windowed_circuit_breaker_configs",
+          "schema": "public"
+        },
+        {
+          "type": "AccountWindowedCircuitBreakerV0",
+          "table": "account_windowed_circuit_breaker_configs",
+          "schema": "public"
+        }
+      ]
+    },
+    {
+      "programId": "hemjuPXBpNvggtaUnN1MwT3wrdhttKEfosTcc2P9Pg8",
+      "accounts": [
+        {
+          "type": "RewardableEntityConfigV0",
+          "table": "rewardable_entity_configs",
+          "schema": "public"
+        },
+        { "type": "MakerV0", "table": "makers", "schema": "public" },
+        {
+          "type": "MakerApprovalV0",
+          "table": "maker_approvals",
+          "schema": "public"
+        },
+        {
+          "type": "KeyToAssetV0",
+          "table": "key_to_assets",
+          "schema": "public"
+        },
+        {
+          "type": "IotHotspotInfoV0",
+          "table": "iot_hotspot_infos",
+          "schema": "public",
+          "plugins": [
+            {
+              "type": "ExtractHexLocation",
+              "config": {
+                "field": "location"
+              }
+            }
+          ]
+        },
+        {
+          "type": "MobileHotspotInfoV0",
+          "table": "mobile_hotspot_infos",
+          "schema": "public",
+          "plugins": [
+            {
+              "type": "ExtractHexLocation",
+              "config": {
+                "field": "location"
+              }
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "programId": "1azyuavdMyvsivtNxPoz6SucD18eDHeXzFCUPq5XU7w",
+      "accounts": [
+        {
+          "type": "LazyDistributorV0",
+          "table": "lazy_distributors",
+          "schema": "public"
+        },
+        { "type": "RecipientV0", "table": "recipients", "schema": "public" }
+      ]
+    }
+  ]
+}

packages/entity-postgres-s3-sink/src/env.ts

@@ -0,0 +1,17 @@
+export const LOOKBACK_HOURS = process.env.LOOKBACK_HOURS
+  ? Number(process.env.LOOKBACK_HOURS)
+  : 25;
+export const AWS_REGION = process.env.AWS_REGION;
+export const MINIO_ENDPOINT = process.env.MINIO_ENDPOINT;
+export const MINIO_ACCESS_KEY = process.env.MINIO_ACCESS_KEY;
+export const MINIO_SECRET_KEY = process.env.MINIO_SECRET_KEY;
+export const S3_BUCKET = process.env.S3_BUCKET!;
+export const CLOUDFRONT_DISTRIBUTION = process.env.CLOUDFRONT_DISTRIBUTION;
+
+// Check for required environment variables
+const requiredEnvVars = ["AWS_REGION", "S3_BUCKET"];
+for (const varName of requiredEnvVars) {
+  if (!process.env[varName]) {
+    throw new Error(`Environment variable ${varName} is required`);
+  }
+}