Skip to content

Commit

Permalink
Implement asset manager uploader and resolver (#2516)
Browse files Browse the repository at this point in the history
## Resolver

Simple form that takes in a `watcloud://...` URI and returns a URL to
the asset.

<img width="874" alt="image"
src="https://github.com/WATonomous/infra-config/assets/5977478/e8c1dcfa-57c5-4eec-8315-513ecc1fdfef">

Success:

<img width="855" alt="image"
src="https://github.com/WATonomous/infra-config/assets/5977478/b853d446-d3f3-44d2-89a9-8a3cbad1de92">

Failure:

<img width="857" alt="image"
src="https://github.com/WATonomous/infra-config/assets/5977478/215aff16-166a-4f4b-8868-1af8295e3d4b">

## Uploader

Frontend for uploading to a public S3 bucket. The target bucket is 
currently hard coded to `https://rgw.watonomous.ca/asset-temp`.

<img width="874" alt="image"
src="https://github.com/WATonomous/infra-config/assets/5977478/b2e8236b-9298-4310-9335-a3700069dfca">

Successful upload:

<img width="872" alt="image"
src="https://github.com/WATonomous/infra-config/assets/5977478/2fd5300d-c5ba-44ae-b722-6c3e59eea829">


Failed upload:

<img width="856" alt="image"
src="https://github.com/WATonomous/infra-config/assets/5977478/35e1d9bf-7aad-48fa-a9ce-c33ac62bc7ad">


### Quirk

**Note:** We have implemented an automated solution for this quirk:
WATonomous/infra-config#2528


The CORS needs to be deployed on rgw (done in code) and on the bucket
(the minio Terraform project doesn't have an option for this). To deploy
the CORS configuration to the bucket manually, do:

```
s3cmd setcors cors.xml s3://asset-temp
```

The `cors.xml` file is provided as an attachment below. It's derived
from https://uppy.io/docs/aws-s3-multipart/#setting-up-your-s3-bucket


[cors.json](https://github.com/WATonomous/infra-config/files/14730054/cors.json)

[cors.xml.txt](https://github.com/WATonomous/infra-config/files/14730057/cors.xml.txt)

Perhaps we can write a custom Terraform
[null_resource](https://chat.openai.com/share/a62408fe-1047-4b26-a726-5658282af010)
that runs a curl:

https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketCors.html

### Notes

The functionality to display a `watcloud://` URI is not yet implemented.
We can do this when we finalize the syntax for the SDK.

Sibling project: WATonomous/infra-config#2411
Parent project: #2306 

### TODOs
- [x] Dark mode: https://uppy.io/docs/dashboard/#theme,
https://nextra.site/docs/docs-theme/api/use-config#return-values
- [x] Set a larger width so that it fills up the space:
https://uppy.io/docs/dashboard/#width
- [x] Show the WATcloud URI after uploading. Perhaps using
https://uppy.io/docs/dashboard/#showlinktofileuploadresult
- [x] Show useful upload errors. (By default it just shows "Not 2xx")
  • Loading branch information
ben-z authored Apr 21, 2024
1 parent e758fae commit 3096117
Show file tree
Hide file tree
Showing 5 changed files with 737 additions and 1 deletion.
235 changes: 235 additions & 0 deletions components/assets.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,235 @@
import Uppy from '@uppy/core';
import type { UppyFile, SuccessResponse } from '@uppy/core'
import { Dashboard } from '@uppy/react';
import AwsS3 from '@uppy/aws-s3';
import { useEffect, useState } from 'react';
import { sha256 } from 'js-sha256';
import { useTheme } from 'nextra-theme-docs';
import { bytesToSize } from '@/lib/utils';
import { Code, Pre } from 'nextra/components';
import { z } from 'zod';
import { useForm } from 'react-hook-form';
import { zodResolver } from '@hookform/resolvers/zod';
import { Button } from "@/components/ui/button";
import {
Form,
FormControl,
FormDescription,
FormField,
FormItem,
FormLabel,
FormMessage,
} from "@/components/ui/form";
import { Input } from "@/components/ui/input";

import '@uppy/core/dist/style.min.css';
import '@uppy/dashboard/dist/style.min.css';

const sha256Cache = new Map<string, string>();

const RESOLVER_URL_PREFIXES = [
"https://rgw.watonomous.ca/asset-perm",
"https://rgw.watonomous.ca/asset-temp",
]

const extractSha256FromURI = (uri: string) => {
const sha256Match = uri.match(/sha256:([a-f0-9]{64})/);
if (!sha256Match) {
throw new Error("Invalid URI: does not contain a SHA-256 hash.");
}
return sha256Match[1];
}

const assetResolverFormSchema = z.object({
uri: z.string(),
});

export function AssetResolver() {
const [isSubmitting, setIsSubmitting] = useState(false);
const [resolvedURL, setResolvedURL] = useState<string>("");
const [errorMessage, setErrorMessage] = useState<string>("");

const form = useForm<z.infer<typeof assetResolverFormSchema>>({
resolver: zodResolver(assetResolverFormSchema),
defaultValues: {
uri: "",
},
});

async function onSubmit({ uri }: z.infer<typeof assetResolverFormSchema>) {
setResolvedURL("");
setErrorMessage("");
setIsSubmitting(true);

try {
if (!uri.startsWith('watcloud://v1/')) {
throw new Error(`Invalid URI: must start with "watcloud://v1/". Got: "${uri}"`);
}

const hash = extractSha256FromURI(uri);

const urls = await Promise.all(RESOLVER_URL_PREFIXES.map(async (prefix) => {
const r = `${prefix}/${hash}`;
const res = await fetch(r, { method: 'HEAD' });
if (res.ok) {
return r;
}
}));

const url = urls.find((url) => url !== undefined);
if (!url) {
throw new Error('Asset not found.');
}

setResolvedURL(url);
} catch (error: any) {
console.error('Error while resolving asset:', error);
setErrorMessage(`Error while resolving asset: ${error.message}`);
}
setIsSubmitting(false);
}

return (
<>
<Form {...form}>
<form onSubmit={form.handleSubmit(onSubmit)} className="space-y-8">
<FormField
control={form.control}
name="uri"
render={({ field }) => (
<FormItem>
<FormLabel>URI</FormLabel>
<FormControl>
<Input placeholder="watcloud://..." {...field} />
</FormControl>
<FormDescription>
The URI of the asset you want to resolve.
</FormDescription>
<FormMessage />
</FormItem>
)}
/>
<Button type="submit" disabled={isSubmitting}>
{isSubmitting ? <>Submitting...</> : <>Submit</>}
</Button>
</form>
</Form>
<h4 className="mt-8 mb-4 text-md">Result</h4>
{resolvedURL && (
<div>
<span className="text-sm text-gray-500">You can access the asset at this URL:</span>
<Pre hasCopyCode className="-mt-5"><Code>{resolvedURL}</Code></Pre>
</div>
)}
{errorMessage && (
<div>
<span className="text-red-500">{errorMessage}</span>
</div>
)}
{!resolvedURL && !errorMessage && (
<p className="text-sm text-gray-500">No result yet. Submit a URI to get started!</p>
)}
</>
);
}


const UPLOADER_MAX_FILE_SIZE = 100 * Math.pow(1024, 2); // Math.pow(1024, 2) = 1MB
const UPLOADER_S3_HOST = 'https://rgw.watonomous.ca';
const UPLOADER_S3_BUCKET = "asset-temp";

export function AssetUploader() {
const { theme } = useTheme();
const uppyTheme = theme === 'dark' ? 'dark' : theme === 'light' ? 'light' : 'auto';

const [successfulUploads, setSuccessfulUploads] = useState<{
name: string;
uri: string;
}[]>([]);
const [errorMessages, setErrorMessages] = useState<string[]>([]);

// IMPORTANT: passing an initializer function to prevent Uppy from being reinstantiated on every render.
const [uppy] = useState(() => new Uppy({
restrictions: {
maxFileSize: UPLOADER_MAX_FILE_SIZE,
},
})
.use(AwsS3, {
shouldUseMultipart: false,
getUploadParameters: async (file) => {
const hash = sha256(await file.data.arrayBuffer());
sha256Cache.set(file.id, hash);

return {
method: 'PUT',
url: `${UPLOADER_S3_HOST}/${UPLOADER_S3_BUCKET}/${hash}`,
};
}
}));

useEffect(() => {
function handleUpload() {
setErrorMessages([]);
}
async function handleUploadSuccess(file: UppyFile | undefined, response: SuccessResponse) {
if (!file) {
console.warn('Got upload success event without a file:', response)
return;
}
const hash = sha256Cache.get(file.id) || sha256(await file.data.arrayBuffer());
const watcloudURI = `watcloud://v1/sha256:${hash}?name=${encodeURIComponent(file.name)}`;
console.log('Uploaded file:', file, 'Response:', response, 'watcloud URI:', watcloudURI);

setSuccessfulUploads((prev) => [{
name: file.name,
uri: watcloudURI,
}, ...prev]);
}
function handleUppyError(file: UppyFile | undefined, error: any) {
console.error('Failed upload:', file, "Error:", error, "Response status:", error.source?.status);
setErrorMessages((prev) => [`Failed to upload ${file?.name}: "${error.message}", response status: "${error.source?.status}", response body: "${error.source?.responseText}"`, ...prev]);
}


uppy.on("upload", handleUpload);
uppy.on('upload-success', handleUploadSuccess);
uppy.on('upload-error', handleUppyError);
return () => {
uppy.off("upload", handleUpload);
uppy.off('upload-success', handleUploadSuccess);
uppy.off('upload-error', handleUppyError);
};
}, [uppy])

return (
<>
<Dashboard
uppy={uppy}
note={`Maximum file size: ${bytesToSize(UPLOADER_MAX_FILE_SIZE, 0)}`}
width="100%"
theme={uppyTheme}
showProgressDetails={true}
/>
<h4 className="mt-8 mb-4 text-md">Successful Uploads</h4>
{successfulUploads.map(({name, uri}) => (
<div key={uri}>
<span className="text-sm text-gray-500">{name}</span>
<Pre hasCopyCode className="-mt-5"><Code>{uri}</Code></Pre>
</div>
))}
{successfulUploads.length === 0 && (
<p className="text-sm text-gray-500">No successful uploads yet. Upload a file to get started!</p>
)}
{errorMessages.length > 0 && (
<>
<h4 className="mt-8 mb-4 text-md">Errors</h4>
{errorMessages.map((message, i) => (
<div key={i}>
<span className="text-red-500">{message}</span>
</div>
))}
</>
)}
</>
);
}
Loading

0 comments on commit 3096117

Please sign in to comment.