rebase package.json

OpenBeta · Feb 17, 2024 · 6d67cd8 · 6d67cd8
1 parent 5bca41d
commit 6d67cd8
Show file tree

Hide file tree

Showing 6 changed files with 163 additions and 77 deletions.
diff --git a/.env b/.env
@@ -17,3 +17,5 @@ TYPESENSE_API_KEY_RW=define_me
 # Auth0
 AUTH0_DOMAIN=https://dev-fmjy7n5n.us.auth0.com
 AUTH0_KID=uciP2tJdJ4BKWoz73Fmln
+
+MAPTILES_WORKING_DIR=./maptiles
diff --git a/package.json b/package.json
@@ -91,9 +91,9 @@
     "export-prod": "./export.sh",
     "prepare": "husky install",
     "import-users": "tsc ; node build/db/utils/jobs/migration/CreateUsersCollection.js",
-    "maptiles:export-db": "tsc ; node build/db/utils/jobs/CragGeojson/index.js",
-    "maptiles:generate": "tsc ; node build/db/utils/jobs/CragGeojson/generate.js",
-    "maptiles:upload": "tsc ; node build/db/utils/jobs/CragGeojson/upload.js"
+    "maptiles:export-db": "tsc ; node build/db/utils/jobs/MapTiles/index.js",
+    "maptiles:generate": "tsc ; node build/db/utils/jobs/MapTiles/generate.js",
+    "maptiles:upload": "tsc ; node build/db/utils/jobs/MapTiles/uploadCmd.js"
   },
   "standard": {
     "plugins": [
@@ -113,4 +113,4 @@
   "engines": {
     "node": ">=16.14.0"
   }
-}
+}
diff --git a/src/db/import/usa/USADay0Seed.ts b/src/db/import/usa/USADay0Seed.ts
@@ -10,7 +10,10 @@ import { logger } from '../../../logger.js'
 const contentDir: string = process.env.CONTENT_BASEDIR ?? ''
 
 const DEFAULT_CONCURRENT_JOBS = 4
-const concurrentJobs: number = process.env.OB_SEED_JOBS !== undefined ? parseInt(process.env.OB_SEED_JOBS) : DEFAULT_CONCURRENT_JOBS
+const concurrentJobs: number =
+  process.env.OB_SEED_JOBS !== undefined
+    ? parseInt(process.env.OB_SEED_JOBS)
+    : DEFAULT_CONCURRENT_JOBS
 
 logger.info('Data dir', contentDir)
 logger.info('Max concurrent jobs: ', concurrentJobs)
@@ -21,7 +24,9 @@ if (contentDir === '') {
 }
 
 const main = async (): Promise<void> => {
-  const limiter = pLimit(concurrentJobs > 0 ? concurrentJobs : DEFAULT_CONCURRENT_JOBS)
+  const limiter = pLimit(
+    concurrentJobs > 0 ? concurrentJobs : DEFAULT_CONCURRENT_JOBS
+  )
 
   // TODO: Allow update.  Right now we drop the entire collection on each run.
   await dropCollection('areas')
@@ -33,17 +38,19 @@ const main = async (): Promise<void> => {
 
   const rootNode = await createRoot('US', 'USA')
 
-  const stats: Array<JobStats | any> = await Promise.all<Array<JobStats | any>>(US_STATES.map(async state => {
-    const code = state.code.toLowerCase()
-    const fRoutes = `${contentDir}/${code}-routes.jsonlines`
-    const fAreas = `${contentDir}/${code}-areas.jsonlines`
+  const stats: Array<JobStats | any> = await Promise.all<Array<JobStats | any>>(
+    US_STATES.map(async state => {
+      const code = state.code.toLowerCase()
+      const fRoutes = `${contentDir}/${code}-routes.jsonlines`
+      const fAreas = `${contentDir}/${code}-areas.jsonlines`
 
-    if (fs.existsSync(fRoutes) && fs.existsSync(fAreas)) {
-      /* eslint-disable-next-line */
-      return limiter(seedState, rootNode, code, fRoutes, fAreas)
-    }
-    return await Promise.resolve()
-  }))
+      if (fs.existsSync(fRoutes) && fs.existsSync(fAreas)) {
+        /* eslint-disable-next-line */
+        return limiter(seedState, rootNode, code, fRoutes, fAreas)
+      }
+      return await Promise.resolve()
+    })
+  )
 
   printStats(stats)
 

diff --git a/src/db/utils/jobs/CragGeojson/index.ts → src/db/utils/jobs/MapTiles/index.ts b/src/db/utils/jobs/CragGeojson/index.ts → src/db/utils/jobs/MapTiles/index.ts
@@ -1,9 +1,21 @@
 import { WriteStream, createWriteStream } from 'node:fs'
-import { point, feature, featureCollection, Feature, Point, Polygon } from '@turf/helpers'
+import {
+  point,
+  feature,
+  featureCollection,
+  Feature,
+  Point,
+  Polygon
+} from '@turf/helpers'
 import os from 'node:os'
 import { MUUID } from 'uuid-mongodb'
 
-import { connectDB, gracefulExit, getAreaModel, getClimbModel } from '../../../index.js'
+import {
+  connectDB,
+  gracefulExit,
+  getAreaModel,
+  getClimbModel
+} from '../../../index.js'
 import { logger } from '../../../../logger.js'
 import { ClimbType } from '../../../ClimbTypes.js'
 import MutableMediaDataSource from '../../../../model/MutableMediaDataSource.js'
@@ -16,14 +28,25 @@ export const WORKING_DIR = './maptiles'
 async function exportLeafCrags (): Promise<void> {
   const model = getAreaModel()
 
-  let features: Array<Feature<Point, {
+  let features: Array<
+  Feature<
+  Point,
+  {
     name: string
-  }>> = []
+  }
+  >
+  > = []
 
   let fileIndex = 0
-  let stream: WriteStream = createWriteStream(`crags.${fileIndex}.geojson`, { encoding: 'utf-8' })
-  const cursor = model.find({ 'metadata.leaf': true, 'metadata.lnglat': { $ne: null } })
-    .populate<{ climbs: ClimbType[] }>({ path: 'climbs', model: getClimbModel() })
+  let stream: WriteStream = createWriteStream(`crags.${fileIndex}.geojson`, {
+    encoding: 'utf-8'
+  })
+  const cursor = model
+    .find({ 'metadata.leaf': true, 'metadata.lnglat': { $ne: null } })
+    .populate<{ climbs: ClimbType[] }>({
+    path: 'climbs',
+    model: getClimbModel()
+  })
     .batchSize(10)
     .allowDiskUse(true)
     .lean()
@@ -33,27 +56,42 @@ async function exportLeafCrags (): Promise<void> {
       continue
     }
 
-    const { metadata, area_name: areaName, pathTokens, ancestors, content, gradeContext, climbs } = doc
+    const {
+      metadata,
+      area_name: areaName,
+      pathTokens,
+      ancestors,
+      content,
+      gradeContext,
+      climbs
+    } = doc
 
     const ancestorArray = ancestors.split(',')
-    const pointFeature = point(doc.metadata.lnglat.coordinates, {
-      id: metadata.area_id,
-      name: areaName,
-      type: 'crag',
-      content,
-      media: await MutableMediaDataSource.getInstance().findMediaByAreaId(metadata.area_id, ancestors),
-      climbs: climbs.map(({ _id, name, type, grades }: ClimbType) => ({
-        id: _id.toUUID().toString(),
-        name,
-        discipline: type,
-        grade: grades
-      })),
-      ancestors: ancestorArray,
-      pathTokens,
-      gradeContext
-    }, {
-      id: metadata.area_id.toUUID().toString()
-    })
+    const pointFeature = point(
+      doc.metadata.lnglat.coordinates,
+      {
+        id: metadata.area_id,
+        name: areaName,
+        type: 'crag',
+        content,
+        media: await MutableMediaDataSource.getInstance().findMediaByAreaId(
+          metadata.area_id,
+          ancestors
+        ),
+        climbs: climbs.map(({ _id, name, type, grades }: ClimbType) => ({
+          id: _id.toUUID().toString(),
+          name,
+          discipline: type,
+          grade: grades
+        })),
+        ancestors: ancestorArray,
+        pathTokens,
+        gradeContext
+      },
+      {
+        id: metadata.area_id.toUUID().toString()
+      }
+    )
     features.push(pointFeature)
 
     if (features.length === 5000) {
@@ -63,7 +101,9 @@ async function exportLeafCrags (): Promise<void> {
       features = []
 
       fileIndex++
-      stream = createWriteStream(`${WORKING_DIR}/crags.${fileIndex}.geojson`, { encoding: 'utf-8' })
+      stream = createWriteStream(`${WORKING_DIR}/crags.${fileIndex}.geojson`, {
+        encoding: 'utf-8'
+      })
     }
   }
 
@@ -105,9 +145,7 @@ async function exportCragGroups (): Promise<void> {
     },
     {
       $match: {
-        $and: [
-          { parentCrags: { $type: 'array', $ne: [] } }
-        ]
+        $and: [{ parentCrags: { $type: 'array', $ne: [] } }]
       }
     },
     {
@@ -147,20 +185,32 @@ async function exportCragGroups (): Promise<void> {
     }
   ])
 
-  const features: Array<Feature<Polygon, {
+  const features: Array<
+  Feature<
+  Polygon,
+  {
     name: string
-  }>> = []
+  }
+  >
+  > = []
 
   for await (const doc of rs) {
-    const polygonFeature = feature(doc.polygon, {
-      type: 'crag-group',
-      id: doc.uuid.toUUID().toString(),
-      name: doc.name,
-      children: doc.childAreaList.map(({ uuid, name, leftRightIndex }) => (
-        { id: uuid.toUUID().toString(), name, lr: leftRightIndex }))
-    }, {
-      id: doc.uuid.toUUID().toString()
-    })
+    const polygonFeature = feature(
+      doc.polygon,
+      {
+        type: 'crag-group',
+        id: doc.uuid.toUUID().toString(),
+        name: doc.name,
+        children: doc.childAreaList.map(({ uuid, name, leftRightIndex }) => ({
+          id: uuid.toUUID().toString(),
+          name,
+          lr: leftRightIndex
+        }))
+      },
+      {
+        id: doc.uuid.toUUID().toString()
+      }
+    )
     features.push(polygonFeature)
   }
 

diff --git a/src/db/utils/jobs/CragGeojson/upload.ts → src/db/utils/jobs/MapTiles/uploadCmd.ts b/src/db/utils/jobs/CragGeojson/upload.ts → src/db/utils/jobs/MapTiles/uploadCmd.ts
@@ -1,11 +1,21 @@
 import fs from 'fs'
-import { PutObjectCommand, PutObjectCommandOutput, S3Client } from '@aws-sdk/client-s3'
-import UploadService, { MapboxUploadCredentials, MapiResponse } from '@mapbox/mapbox-sdk/services/uploads.js'
+import { config } from 'dotenv'
+import {
+  PutObjectCommand,
+  PutObjectCommandOutput,
+  S3Client
+} from '@aws-sdk/client-s3'
+import UploadService, {
+  MapboxUploadCredentials,
+  MapiResponse
+} from '@mapbox/mapbox-sdk/services/uploads.js'
 import { logger } from '../../../../logger.js'
-import { WORKING_DIR } from './index.js'
+
+config({ path: ['.env.local', '.env'] })
 
 const mapboxUsername = process.env.MAPBOX_USERNAME
 const mapboxToken = process.env.MAPBOX_TOKEN
+const workingDir = process.env.MAPTILES_WORKING_DIR
 
 if (mapboxUsername == null) {
   throw new Error('MAPBOX_USERNAME not set')
@@ -15,6 +25,10 @@ if (mapboxToken == null) {
   throw new Error('MAPBOX_TOKEN not set')
 }
 
+if (workingDir == null) {
+  throw new Error('MAPTILES_WORKING_DIR not set')
+}
+
 const uploadsClient = UploadService({ accessToken: mapboxToken })
 
 const getCredentials = async (): Promise<MapboxUploadCredentials> => {
@@ -24,7 +38,10 @@ const getCredentials = async (): Promise<MapboxUploadCredentials> => {
     .then(response => response.body)
 }
 
-const stageFileOnS3 = async (credentials: MapboxUploadCredentials, filePath: string): Promise<PutObjectCommandOutput> => {
+const stageFileOnS3 = async (
+  credentials: MapboxUploadCredentials,
+  filePath: string
+): Promise<PutObjectCommandOutput> => {
   const s3Client = new S3Client({
     region: 'us-east-1',
     credentials: {
@@ -58,12 +75,17 @@ interface UploadOptions {
   filePath: string
 }
 
-const notifyMapbox = async (credentials: MapboxUploadCredentials, { tilesetId, name }: UploadOptions): Promise<MapiResponse> => {
-  const res = await uploadsClient.createUpload({
-    tileset: `${mapboxUsername}.${tilesetId}`,
-    url: credentials.url,
-    name
-  }).send()
+const notifyMapbox = async (
+  credentials: MapboxUploadCredentials,
+  { tilesetId, name }: UploadOptions
+): Promise<MapiResponse> => {
+  const res = await uploadsClient
+    .createUpload({
+      tileset: `${mapboxUsername}.${tilesetId}`,
+      url: credentials.url,
+      name
+    })
+    .send()
   return res
 }
 
@@ -75,20 +97,25 @@ const notifyMapbox = async (credentials: MapboxUploadCredentials, { tilesetId, n
 export const upload = async (options: UploadOptions): Promise<void> => {
   try {
     const credentials = await getCredentials()
-    await stageFileOnS3(credentials, options.filePath)
-    logger.info('File staged on S3')
+    const ret = await stageFileOnS3(credentials, options.filePath)
+    logger.info(ret.$metadata, 'File staged on S3')
     const res = await notifyMapbox(credentials, options)
-    if ((res.statusCode >= 200 && res.statusCode < 300)) {
+    if (res.statusCode >= 200 && res.statusCode < 300) {
       logger.info('File uploaded to Mapbox')
       return await Promise.resolve()
     }
-    throw new Error(`Create upload failed with status code ${res.statusCode as string}`)
+    throw new Error(
+      `Create upload failed with status code ${res.statusCode as string}`
+    )
   } catch (err) {
-    logger.error(err)
+    logger.error(err, 'Failed to upload file to Mapbox')
   }
 }
 
 export const uploadCragsTiles = async (): Promise<void> => {
-  const filePath = `${WORKING_DIR}/crags.mbtiles`
+  logger.info('Uploading crags tiles')
+  const filePath = `${workingDir}/crags.mbtiles`
   await upload({ tilesetId: 'crags', name: 'all crags and boulders', filePath })
 }
+
+await uploadCragsTiles()
diff --git a/yarn.lock b/yarn.lock
@@ -4491,10 +4491,10 @@ dot-object@^2.1.4:
     commander "^4.0.0"
     glob "^7.1.5"
 
-dotenv@^10.0.0:
-  version "10.0.0"
-  resolved "https://registry.npmjs.org/dotenv/-/dotenv-10.0.0.tgz"
-  integrity sha512-rlBi9d8jpv9Sf1klPjNfFAuWDjKLwTIJJ/VxtoTwIR6hnZxcEOQCZg2oIL3MWBYw5GpUDKOEnND7LXTbIpQ03Q==
+dotenv@^16.4.2:
+  version "16.4.2"
+  resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.4.2.tgz#7ca798f89ae2011846bbdbf6470785307754120d"
+  integrity sha512-rZSSFxke7d9nYQ5NeMIwp5PP+f8wXgKNljpOb7KtH6SKW1cEqcXAz9VSJYVLKe7Jhup/gUYOkaeSVyK8GJ+nBg==
 
 duplexify@^4.0.0, duplexify@^4.1.1:
   version "4.1.2"