feat: upload files from cli

package.json

@@ -31,6 +31,7 @@
     "cosmiconfig": "^9.0.0",
     "dotenv": "^16.4.5",
     "exif-parser": "^0.1.12",
+    "glob": "^10.4.3",
     "sharp": "^0.33.4",
     "tsx": "^4.11.2",
     "winston": "^3.13.0",

rollup.config.js

@@ -28,5 +28,6 @@ export default {
     "sharp",
     "winston",
     "zod",
+    "glob",
   ],
 };

src/cli.ts

@@ -5,6 +5,7 @@ import { Command } from "commander";
 import { checkState } from "./check";
 import { run } from "./run";
 import { deletePath } from "./delete";
+import { uploadPath } from "./file";
 
 const program = new Command();
 
@@ -34,6 +35,24 @@ program
     await checkState(output);
   });
 
+program
+  .command("upload <src> <dest>")
+  .option("-s, --show-files", "show files")
+  .option("-f, --force", "skip confirmation")
+  .description(
+    "upload all files from src directory to dest in the remote bucket"
+  )
+  .action(
+    async (
+      src: string,
+      dest: string,
+      options: { showFiles?: boolean; force?: boolean }
+    ) => {
+      const { showFiles, force } = options;
+      await uploadPath(src, dest, { showFiles, force });
+    }
+  );
+
 program
   .command("delete <path>")
   .option("-s, --show-files", "show files")

src/file.ts

@@ -0,0 +1,91 @@
+import * as fs from "fs/promises";
+import ansis from "ansis";
+import { confirm } from "@inquirer/prompts";
+import { glob } from "glob";
+
+import { logger } from "./logger";
+import { getConfig } from "./config";
+import { uploadFilesBulk } from "./s3";
+
+type UploadPathOptions = {
+  force?: boolean;
+  showFiles?: boolean;
+};
+
+export const uploadPath = async (
+  src: string,
+  dest: string,
+  options: UploadPathOptions
+) => {
+  const { force = false, showFiles = false } = options;
+  const config = getConfig();
+  const { bucketName } = config;
+
+  const { filePaths, totalSize } = await findFilesAndSize(src);
+
+  if (filePaths.length === 0) {
+    logger.info("No files to upload from that path");
+    return;
+  }
+
+  logger.info("Number of items to upload: %o", filePaths.length);
+  logger.info("Total size to upload: %s", formatBytes(totalSize));
+
+  if (showFiles) {
+    for (const filePath of filePaths) {
+      logger.info(ansis.gray(` - ${filePath}`));
+    }
+  }
+
+  if (!force) {
+    const confirmUpload = await confirm({
+      message: `Are you sure you want to upload ${
+        filePaths.length
+      } items (${formatBytes(totalSize)})?`,
+    });
+
+    if (!confirmUpload) {
+      logger.info("Upload aborted by the user.");
+      return;
+    }
+  }
+
+  try {
+    await uploadFilesBulk(bucketName, src, dest, filePaths);
+    logger.info(ansis.green("Done!"));
+  } catch (error) {
+    logger.error("Error occurred while uploading files:", error);
+  }
+};
+
+async function findFilesAndSize(
+  src: string
+): Promise<{ filePaths: string[]; totalSize: number }> {
+  const filePaths = await glob("**/*", {
+    cwd: src,
+    nodir: true,
+    absolute: true,
+  });
+
+  let totalSize = 0;
+  await Promise.all(
+    filePaths.map(async (filePath) => {
+      const stats = await fs.stat(filePath);
+      totalSize += stats.size;
+    })
+  );
+
+  return { filePaths, totalSize };
+}
+
+export function formatBytes(bytes: number, decimals = 2) {
+  if (bytes === 0) return "0 Bytes";
+
+  const k = 1024;
+  const dm = decimals < 0 ? 0 : decimals;
+  const sizes = ["Bytes", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"];
+
+  const i = Math.floor(Math.log(bytes) / Math.log(k));
+
+  return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + " " + sizes[i];
+}

src/image.ts

@@ -6,7 +6,8 @@ import sharp from "sharp";
 import ExifParser from "exif-parser";
 
 import { logger } from "./logger";
-import { deleteFilesBulk, downloadImage, getClient, uploadImage } from "./s3";
+import { downloadImage, getClient, uploadImage } from "./s3";
+import { deriveContentType } from "./utils";
 import {
   type Ext,
   type ImageToCreate,
@@ -195,25 +196,3 @@ const parseExifData = async (imgData: Buffer): Promise<number | null> => {
     return null;
   }
 };
-
-const deriveContentType = (format: string | undefined): string | undefined => {
-  switch (format?.toLowerCase()) {
-    case "jpeg":
-    case "jpg":
-      return "image/jpeg";
-    case "png":
-      return "image/png";
-    case "webp":
-      return "image/webp";
-    case "gif":
-      return "image/gif";
-    case "svg":
-      return "image/svg+xml";
-    case "tiff":
-      return "image/tiff";
-    case "avif":
-      return "image/avif";
-    default:
-      return undefined;
-  }
-};

src/s3.ts

@@ -1,3 +1,5 @@
+import * as path from "path";
+import * as fs from "fs/promises";
 import type { Readable } from "stream";
 import Bottleneck from "bottleneck";
 import ansis from "ansis";
@@ -15,6 +17,7 @@ import type {
 
 import { getConfig } from "./config";
 import { logger } from "./logger";
+import { deriveContentType } from "./utils";
 
 export const createS3Client = (
   accessKeyId: string,
@@ -165,3 +168,63 @@ export const deleteFile = async (
   const command = new DeleteObjectCommand(deleteParams);
   await s3Client.send(command);
 };
+
+export const uploadFilesBulk = async (
+  bucketName: string,
+  srcPath: string,
+  destPath: string,
+  filePaths: string[]
+) => {
+  const limiter = new Bottleneck({
+    maxConcurrent: 25,
+  });
+
+  const s3Client = await getClient();
+
+  const uploadTasks = filePaths.map((filePath, index) =>
+    limiter.schedule(() => {
+      const uploadWithLog = async () => {
+        const relativePath = path.relative(srcPath, filePath);
+        const s3Key = path.join(destPath, relativePath).replace(/\\/g, "/");
+        logger.info(
+          ansis.gray(
+            `${index + 1}/${
+              filePaths.length
+            }: uploading ${filePath} to ${bucketName}/${s3Key}`
+          )
+        );
+        return await uploadFile(bucketName, s3Client, filePath, s3Key);
+      };
+
+      return uploadWithLog();
+    })
+  );
+
+  await Promise.all(uploadTasks);
+};
+
+const uploadFile = async (
+  bucketName: string,
+  s3Client: S3Client,
+  filePath: string,
+  s3Key: string
+) => {
+  try {
+    const fileContent = await fs.readFile(filePath);
+    const fileExtension = path.extname(filePath).toLowerCase().slice(1);
+    const contentType = deriveContentType(fileExtension);
+
+    const params = {
+      Bucket: bucketName,
+      Key: s3Key,
+      Body: fileContent,
+      ContentType: contentType,
+    };
+
+    const command = new PutObjectCommand(params);
+    await s3Client.send(command);
+  } catch (error) {
+    logger.error(`Error uploading file ${filePath}:`, error);
+    throw error;
+  }
+};

src/utils.ts

@@ -0,0 +1,24 @@
+export const deriveContentType = (format: string | undefined) => {
+  switch (format?.toLowerCase()) {
+    case "jpeg":
+    case "jpg":
+      return "image/jpeg";
+    case "png":
+      return "image/png";
+    case "webp":
+      return "image/webp";
+    case "gif":
+      return "image/gif";
+    case "svg":
+      return "image/svg+xml";
+    case "tiff":
+    case "tif":
+      return "image/tiff";
+    case "avif":
+      return "image/avif";
+    case "pdf":
+      return "application/pdf";
+    default:
+      return "application/octet-stream";
+  }
+};