Skip to content

Commit

Permalink
[bug] content-publisher needs to be its own source of truth for annou…
Browse files Browse the repository at this point in the history
…ncement attachments (#341)

# Description
As a user, I should not have to keep track of the association between an
uploaded asset reference ID and its media type. More specifically, I
should not be able to tell content-publisher the type of an asset in an
Announcment publish request (Broadcast, Reply, etc), as that opens up an
attack vector to specify a media type that does not match the actual
uploaded asset. content-publisher should be its own source of truth in
coordinating asset type with the actual asset.

- [x] Remove `type` from `BroadcastDto`, `UpdateDto`, `ReplyDto`
   -  Removed from `AssetDto`, which is included by all of the above
- [x] Update OpenAPI spec
   -  Updated using the `generate` scripts 
- [x] Add `type` to cached metadata for uploaded asset

Closes #338
  • Loading branch information
mattheworris authored Aug 5, 2024
1 parent f3dc764 commit a166a97
Show file tree
Hide file tree
Showing 8 changed files with 187 additions and 163 deletions.
13 changes: 1 addition & 12 deletions services/content-publishing/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -167,27 +167,16 @@ Use the provided [env.template](./env.template) file to create an initial enviro

```sh
cp env.template .env
cp env.template .env.docker.dev
```

2. Configure the environment variable values according to your environment.

### Setup

Clone this repository to your desired folder:

Example commands:

```sh
git clone [email protected]:AmplicaLabs/content-publishing-service.git
cd content-publishing-service
```

### Install

Install NPM Dependencies:

```sh
cd services/content-publishing
npm install
```

Expand Down
66 changes: 31 additions & 35 deletions services/content-publishing/apps/api/src/api.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,8 @@ import { BulkJobOptions } from 'bullmq/dist/esm/interfaces';
import { InjectRedis } from '@songkeys/nestjs-redis';
import Redis from 'ioredis';
import { HttpErrorByCode } from '@nestjs/common/utils/http-error-by-code.util';
import {
AnnouncementTypeDto,
RequestTypeDto,
AnnouncementResponseDto,
AssetIncludedRequestDto,
isImage,
UploadResponseDto,
} from '#libs/dtos';
import { IRequestJob, IAssetMetadata, IAssetJob } from '#libs/interfaces';
import { AnnouncementTypeDto, RequestTypeDto, AnnouncementResponseDto, AssetIncludedRequestDto, isImage, UploadResponseDto, AttachmentType } from '#libs/dtos';
import { IRequestJob, IAssetMetadata, IAssetJob, IAssetTypeInfo } from '#libs/interfaces';
import { REQUEST_QUEUE_NAME, ASSET_QUEUE_NAME } from '#libs/queues/queue.constants';
import { calculateIpfsCID } from '#libs/utils/ipfs';
import { getAssetMetadataKey, getAssetDataKey, STORAGE_EXPIRE_UPPER_LIMIT_SECONDS } from '#libs/utils/redis';
Expand All @@ -35,7 +28,7 @@ export class ApiService {
announcementType: AnnouncementTypeDto,
dsnpUserId: string,
content: RequestTypeDto,
assetToMimeType?: Map<string, string>,
assetToMimeType?: IRequestJob['assetToMimeType'],
): Promise<AnnouncementResponseDto> {
const data = {
content,
Expand All @@ -60,12 +53,10 @@ export class ApiService {
};
}

async validateAssetsAndFetchMetadata(content: AssetIncludedRequestDto): Promise<Map<string, string> | undefined> {
async validateAssetsAndFetchMetadata(content: AssetIncludedRequestDto): Promise<IRequestJob['assetToMimeType'] | undefined> {
const checkingList: { onlyImage: boolean; referenceId: string }[] = [];
if (content.profile) {
content.profile.icon?.forEach((reference) =>
checkingList.push({ onlyImage: true, referenceId: reference.referenceId }),
);
content.profile.icon?.forEach((reference) => checkingList.push({ onlyImage: true, referenceId: reference.referenceId }));
} else if (content.content) {
content.content.assets?.forEach((asset) =>
asset.references?.forEach((reference) =>
Expand All @@ -77,19 +68,15 @@ export class ApiService {
);
}

const redisResults = await Promise.all(
checkingList.map((obj) => this.redis.get(getAssetMetadataKey(obj.referenceId))),
);
const redisResults = await Promise.all(checkingList.map((obj) => this.redis.get(getAssetMetadataKey(obj.referenceId))));
const errors: string[] = [];
const map = new Map();
redisResults.forEach((res, index) => {
if (res === null) {
errors.push(
`${content.profile ? 'profile.icon' : 'content.assets'}.referenceId ${checkingList[index].referenceId} does not exist!`,
);
errors.push(`${content.profile ? 'profile.icon' : 'content.assets'}.referenceId ${checkingList[index].referenceId} does not exist!`);
} else {
const metadata: IAssetMetadata = JSON.parse(res);
map[checkingList[index].referenceId] = metadata.mimeType;
map[checkingList[index].referenceId] = { mimeType: metadata.mimeType, attachmentType: metadata.type };

// checks if attached asset is an image
if (checkingList[index].onlyImage && !isImage(metadata.mimeType)) {
Expand All @@ -114,20 +101,29 @@ export class ApiService {
const jobs: any[] = [];

Check warning on line 101 in services/content-publishing/apps/api/src/api.service.ts

View workflow job for this annotation

GitHub Actions / [content-publishing] Build and Test

Unexpected any. Specify a different type
files.forEach((f, index) => {
// adding data and metadata to the transaction
dataTransaction = dataTransaction.setex(
getAssetDataKey(references[index]),
STORAGE_EXPIRE_UPPER_LIMIT_SECONDS,
f.buffer,
);
metadataTransaction = metadataTransaction.setex(
getAssetMetadataKey(references[index]),
STORAGE_EXPIRE_UPPER_LIMIT_SECONDS,
JSON.stringify({
ipfsCid: references[index],
mimeType: f.mimetype,
createdOn: Date.now(),
} as IAssetMetadata),
);
dataTransaction = dataTransaction.setex(getAssetDataKey(references[index]), STORAGE_EXPIRE_UPPER_LIMIT_SECONDS, f.buffer);
const type = ((m) => {
switch (m) {
case 'image':
return AttachmentType.IMAGE;
case 'audio':
return AttachmentType.AUDIO;
case 'video':
return AttachmentType.VIDEO;
default:
throw new Error('Invalid MIME type');
}
})(f.mimetype.split('/')[0]);

const assetCache: IAssetMetadata = {
ipfsCid: references[index],
mimeType: f.mimetype,
createdOn: Date.now(),
type: type,
};

metadataTransaction = metadataTransaction.setex(getAssetMetadataKey(references[index]), STORAGE_EXPIRE_UPPER_LIMIT_SECONDS, JSON.stringify(assetCache));

// adding asset job to the jobs
jobs.push({
name: `Asset Job - ${references[index]}`,
Expand Down
97 changes: 91 additions & 6 deletions services/content-publishing/apps/api/src/metadata.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,93 @@
/* eslint-disable */
export default async () => {
const t = {
["../../../libs/common/src/dtos/activity.dto"]: await import("../../../libs/common/src/dtos/activity.dto"),
["../../../libs/common/src/dtos/announcement.dto"]: await import("../../../libs/common/src/dtos/announcement.dto")
};
return { "@nestjs/swagger": { "models": [[import("../../../libs/common/src/dtos/common.dto"), { "DsnpUserIdParam": { userDsnpId: { required: true, type: () => String } }, "AnnouncementResponseDto": { referenceId: { required: true, type: () => String } }, "UploadResponseDto": { assetIds: { required: true, type: () => [String] } }, "FilesUploadDto": { files: { required: true, type: () => [Object] } } }], [import("../../../libs/common/src/dtos/activity.dto"), { "LocationDto": { name: { required: true, type: () => String, minLength: 1 }, accuracy: { required: false, type: () => Number, minimum: 0, maximum: 100 }, altitude: { required: false, type: () => Number }, latitude: { required: false, type: () => Number }, longitude: { required: false, type: () => Number }, radius: { required: false, type: () => Number, minimum: 0 }, units: { required: false, enum: t["../../../libs/common/src/dtos/activity.dto"].UnitTypeDto } }, "AssetReferenceDto": { referenceId: { required: true, type: () => String, minLength: 1 }, height: { required: false, type: () => Number, minimum: 1 }, width: { required: false, type: () => Number, minimum: 1 }, duration: { required: false, type: () => String, pattern: "DURATION_REGEX" } }, "TagDto": { type: { required: true, enum: t["../../../libs/common/src/dtos/activity.dto"].TagTypeDto }, name: { required: false, type: () => String, minLength: 1 }, mentionedId: { required: false, type: () => String } }, "AssetDto": { type: { required: true, enum: t["../../../libs/common/src/dtos/activity.dto"].AttachmentTypeDto }, references: { required: false, type: () => [t["../../../libs/common/src/dtos/activity.dto"].AssetReferenceDto] }, name: { required: false, type: () => String, minLength: 1 }, href: { required: false, type: () => String, minLength: 1 } }, "BaseActivityDto": { name: { required: false, type: () => String }, tag: { required: false, type: () => [t["../../../libs/common/src/dtos/activity.dto"].TagDto] }, location: { required: false, type: () => t["../../../libs/common/src/dtos/activity.dto"].LocationDto } }, "NoteActivityDto": { content: { required: true, type: () => String, minLength: 1 }, published: { required: true, type: () => String }, assets: { required: false, type: () => [t["../../../libs/common/src/dtos/activity.dto"].AssetDto] } }, "ProfileActivityDto": { icon: { required: false, type: () => [t["../../../libs/common/src/dtos/activity.dto"].AssetReferenceDto] }, summary: { required: false, type: () => String }, published: { required: false, type: () => String } } }], [import("../../../libs/common/src/dtos/announcement.dto"), { "BroadcastDto": { content: { required: true, type: () => t["../../../libs/common/src/dtos/activity.dto"].NoteActivityDto } }, "ReplyDto": { inReplyTo: { required: true, type: () => String }, content: { required: true, type: () => t["../../../libs/common/src/dtos/activity.dto"].NoteActivityDto } }, "TombstoneDto": { targetContentHash: { required: true, type: () => String }, targetAnnouncementType: { required: true, enum: t["../../../libs/common/src/dtos/announcement.dto"].ModifiableAnnouncementTypeDto } }, "UpdateDto": { targetContentHash: { required: true, type: () => String }, targetAnnouncementType: { required: true, enum: t["../../../libs/common/src/dtos/announcement.dto"].ModifiableAnnouncementTypeDto }, content: { required: true, type: () => t["../../../libs/common/src/dtos/activity.dto"].NoteActivityDto } }, "ReactionDto": { emoji: { required: true, type: () => String, minLength: 1, pattern: "DSNP_EMOJI_REGEX" }, apply: { required: true, type: () => Number, minimum: 0, maximum: 255 }, inReplyTo: { required: true, type: () => String } }, "ProfileDto": { profile: { required: true, type: () => t["../../../libs/common/src/dtos/activity.dto"].ProfileActivityDto } } }]], "controllers": [[import("./controllers/health.controller"), { "HealthController": { "healthz": {}, "livez": {}, "readyz": {} } }]] } };
};
const t = {
['../../../libs/common/src/dtos/activity.dto']: await import('../../../libs/common/src/dtos/activity.dto'),
['../../../libs/common/src/dtos/announcement.dto']: await import('../../../libs/common/src/dtos/announcement.dto'),
};
return {
'@nestjs/swagger': {
models: [
[
import('../../../libs/common/src/dtos/common.dto'),
{
DsnpUserIdParam: { userDsnpId: { required: true, type: () => String } },
AnnouncementResponseDto: { referenceId: { required: true, type: () => String } },
UploadResponseDto: { assetIds: { required: true, type: () => [String] } },
FilesUploadDto: { files: { required: true, type: () => [Object] } },
},
],
[
import('../../../libs/common/src/dtos/activity.dto'),
{
LocationDto: {
name: { required: true, type: () => String, minLength: 1 },
accuracy: { required: false, type: () => Number, minimum: 0, maximum: 100 },
altitude: { required: false, type: () => Number },
latitude: { required: false, type: () => Number },
longitude: { required: false, type: () => Number },
radius: { required: false, type: () => Number, minimum: 0 },
units: { required: false, enum: t['../../../libs/common/src/dtos/activity.dto'].UnitTypeDto },
},
AssetReferenceDto: {
referenceId: { required: true, type: () => String, minLength: 1 },
height: { required: false, type: () => Number, minimum: 1 },
width: { required: false, type: () => Number, minimum: 1 },
duration: { required: false, type: () => String, pattern: 'DURATION_REGEX' },
},
TagDto: {
type: { required: true, enum: t['../../../libs/common/src/dtos/activity.dto'].TagTypeDto },
name: { required: false, type: () => String, minLength: 1 },
mentionedId: { required: false, type: () => String },
},
AssetDto: {
references: { required: false, type: () => [t['../../../libs/common/src/dtos/activity.dto'].AssetReferenceDto] },
name: { required: false, type: () => String, minLength: 1 },
href: { required: false, type: () => String, minLength: 1 },
},
BaseActivityDto: {
name: { required: false, type: () => String },
tag: { required: false, type: () => [t['../../../libs/common/src/dtos/activity.dto'].TagDto] },
location: { required: false, type: () => t['../../../libs/common/src/dtos/activity.dto'].LocationDto },
},
NoteActivityDto: {
content: { required: true, type: () => String, minLength: 1 },
published: { required: true, type: () => String },
assets: { required: false, type: () => [t['../../../libs/common/src/dtos/activity.dto'].AssetDto] },
},
ProfileActivityDto: {
icon: { required: false, type: () => [t['../../../libs/common/src/dtos/activity.dto'].AssetReferenceDto] },
summary: { required: false, type: () => String },
published: { required: false, type: () => String },
},
},
],
[
import('../../../libs/common/src/dtos/announcement.dto'),
{
BroadcastDto: { content: { required: true, type: () => t['../../../libs/common/src/dtos/activity.dto'].NoteActivityDto } },
ReplyDto: {
inReplyTo: { required: true, type: () => String },
content: { required: true, type: () => t['../../../libs/common/src/dtos/activity.dto'].NoteActivityDto },
},
TombstoneDto: {
targetContentHash: { required: true, type: () => String },
targetAnnouncementType: { required: true, enum: t['../../../libs/common/src/dtos/announcement.dto'].ModifiableAnnouncementTypeDto },
},
UpdateDto: {
targetContentHash: { required: true, type: () => String },
targetAnnouncementType: { required: true, enum: t['../../../libs/common/src/dtos/announcement.dto'].ModifiableAnnouncementTypeDto },
content: { required: true, type: () => t['../../../libs/common/src/dtos/activity.dto'].NoteActivityDto },
},
ReactionDto: {
emoji: { required: true, type: () => String, minLength: 1, pattern: 'DSNP_EMOJI_REGEX' },
apply: { required: true, type: () => Number, minimum: 0, maximum: 255 },
inReplyTo: { required: true, type: () => String },
},
ProfileDto: { profile: { required: true, type: () => t['../../../libs/common/src/dtos/activity.dto'].ProfileActivityDto } },
},
],
],
controllers: [[import('./controllers/health.controller'), { HealthController: { healthz: {}, livez: {}, readyz: {} } }]],
},
};
};
Loading

0 comments on commit a166a97

Please sign in to comment.