Skip to content

Commit

Permalink
Merge branch 'main' into 267-all-services-notification-queue-should-u…
Browse files Browse the repository at this point in the history 
…npause-based-on-capacity-refilled-event
  • Loading branch information
@JoeCap08055
JoeCap08055 committed Aug 9, 2024
2 parents 8b372a5 + c4372bf commit fa91f25
Show file tree
Hide file tree
Showing 19 changed files with 142 additions and 65 deletions.
1 change: 1 addition & 0 deletions services/account/ENVIRONMENT.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ This application recognizes the following environment variables:
| `API_PORT` | HTTP port that the application listens on | 1025 - 65535 | | 3000 |
| `BLOCKCHAIN_SCAN_INTERVAL_SECONDS` | How many seconds to delay between successive scans of the chain for new content (after end of chain is reached) | > 0 | | 12 |
| `TRUST_UNFINALIZED_BLOCKS` | Whether to examine blocks that have not been finalized when tracking extrinsic completion | boolean | | false |
| `CACHE_KEY_PREFIX` | Prefix to use for Redis cache keys | string | | account: |
| `CAPACITY_LIMIT` | Maximum amount of provider capacity this app is allowed to use (per epoch) type: 'percentage' 'amount' value: number (may be percentage, ie '80', or absolute amount of capacity) | JSON [(example)](./env.template) | Y | |
| `FREQUENCY_URL` | Blockchain node address | http(s): or ws(s): URL | Y | |
| `FREQUENCY_HTTP_URL` | Blockchain node address resolvable from the client browser | http(s): URL | Y | |
Expand Down
1 change: 1 addition & 0 deletions services/account/apps/api/src/api.module.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ import { ApiService, AccountsService, HandlesService, DelegationService, KeysSer
config: [
{
url: configService.redisUrl.toString(),
keyPrefix: configService.cacheKeyPrefix,
maxRetriesPerRequest: null,
onClientCreated(client) {
redisEventsToEventEmitter(client, eventEmitter);
Expand Down
1 change: 1 addition & 0 deletions services/account/apps/worker/src/worker.module.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ import { TransactionPublisherModule } from './transaction_publisher/publisher.mo
config: [
{
url: configService.redisUrl.toString(),
keyPrefix: configService.cacheKeyPrefix,
maxRetriesPerRequest: null,
onClientCreated(client) {
redisEventsToEventEmitter(client, eventEmitter);
Expand Down
3 changes: 3 additions & 0 deletions services/account/env.template
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,3 +66,6 @@ SIWF_DOMAIN=localhost

# Enable debug mode for development
DEBUG=true

# Prefix to use for Redis cache keys
CACHE_KEY_PREFIX=account:
5 changes: 5 additions & 0 deletions services/account/libs/common/src/config/config.service.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ describe('AccountSericeConfig', () => {
HEALTH_CHECK_MAX_RETRY_INTERVAL_SECONDS: undefined,
HEALTH_CHECK_MAX_RETRIES: undefined,
CAPACITY_LIMIT: undefined,
CACHE_KEY_PREFIX: undefined,
};

beforeAll(() => {
Expand Down Expand Up @@ -222,5 +223,9 @@ describe('AccountSericeConfig', () => {
it('should get capacity limit', () => {
expect(JSON.stringify(accountServiceConfig.capacityLimit)).toStrictEqual(ALL_ENV.CAPACITY_LIMIT!);
});

it('should get cache key prefix', () => {
expect(accountServiceConfig.cacheKeyPrefix).toStrictEqual(ALL_ENV.CACHE_KEY_PREFIX?.toString());
});
});
});
5 changes: 5 additions & 0 deletions services/account/libs/common/src/config/config.service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ export interface ConfigEnvironmentVariables {
HEALTH_CHECK_MAX_RETRY_INTERVAL_SECONDS: number;
HEALTH_CHECK_MAX_RETRIES: number;
CAPACITY_LIMIT: string;
CACHE_KEY_PREFIX: string;
}

/// Config service to get global app and provider-specific config values.
Expand All @@ -46,6 +47,10 @@ export class ConfigService {
}
}

public get cacheKeyPrefix(): string {
return this.nestConfigService.get('CACHE_KEY_PREFIX')!;
}

public get blockchainScanIntervalSeconds(): number {
return this.nestConfigService.get<number>('BLOCKCHAIN_SCAN_INTERVAL_SECONDS') ?? 12;
}
Expand Down
1 change: 1 addition & 0 deletions services/account/libs/common/src/config/env.config.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ const capacityLimitsSchema = Joi.object({
export const configModuleOptions: ConfigModuleOptions = {
isGlobal: true,
validationSchema: Joi.object({
CACHE_KEY_PREFIX: Joi.string().default('account:'),
BLOCKCHAIN_SCAN_INTERVAL_SECONDS: Joi.number().min(1).default(12),
TRUST_UNFINALIZED_BLOCKS: Joi.bool().default(false),
REDIS_URL: Joi.string().uri().required(),
Expand Down
1 change: 1 addition & 0 deletions services/account/libs/common/src/queues/queues.module.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ export class QueueModule {
imports: [ConfigModule],
useFactory: (configService: ConfigService) => ({
connection: new Redis(configService.redisUrl.toString(), redisOptions || {}),
prefix: `${configService.cacheKeyPrefix}:bull`,
}),
inject: [ConfigService],
}),
Expand Down
33 changes: 17 additions & 16 deletions services/content-publishing/ENVIRONMENT.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,20 +2,21 @@

This application recognizes the following environment variables:

| Name | Description | Range/Type | Required? | Default |
| ----------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :------------------------------: | :-------: | :-----: |
| `API_PORT` | HTTP port that the application listens on | 1025 - 65535 | | 3000 |
| `CAPACITY_LIMIT` | Maximum amount of provider capacity this app is allowed to use (per epoch) type: 'percentage' 'amount' value: number (may be percentage, ie '80', or absolute amount of capacity) | JSON [(example)](./env.template) | Y | |
| `FREQUENCY_URL` | Blockchain node address | http(s): or ws(s): URL | Y | |
| `PROVIDER_ACCOUNT_SEED_PHRASE` | Seed phrase for provider MSA control key | string | Y | |
| `PROVIDER_ID` | Provider MSA ID | integer | Y | |
| Name | Description | Range/Type | Required? | Default |
| ----------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :------------------------------: | :-------: | :-------------------------: |
| `API_PORT` | HTTP port that the application listens on | 1025 - 65535 | | 3000 |
| `CACHE_KEY_PREFIX` | Prefix to use for Redis cache keys | string | | content-publishing-service: |
| `CAPACITY_LIMIT` | Maximum amount of provider capacity this app is allowed to use (per epoch) type: 'percentage' 'amount' value: number (may be percentage, ie '80', or absolute amount of capacity) | JSON [(example)](./env.template) | Y | |
| `FREQUENCY_URL` | Blockchain node address | http(s): or ws(s): URL | Y | |
| `PROVIDER_ACCOUNT_SEED_PHRASE` | Seed phrase for provider MSA control key | string | Y | |
| `PROVIDER_ID` | Provider MSA ID | integer | Y | |
| `REDIS_URL` | Connection URL for Redis | URL | Y |
| `ASSET_EXPIRATION_INTERVAL_SECONDS` | Number of seconds to keep completed asset entrie in the cache before expiring them | > 0 | Y | |
| `ASSET_UPLOAD_VERIFICATION_DELAY_SECONDS` | Base delay in seconds used for exponential backoff while waiting for uploaded assets to be verified available before publishing a content notice | >= 0 | Y | |
| `BATCH_INTERVAL_SECONDS` | Number of seconds between content publishing batches. This is so that the service waits a reasonable amount of time for additional content to publishing before submitting a batch--it represents a trade-off between maximum batch fullness and minimal wait time for published content | > 0 | Y | |
| `BATCH_MAX_COUNT` | Maximum number of items that can be submitted in a single batch | > 0 | Y | |
| `FILE_UPLOAD_MAX_SIZE_IN_BYTES` | Max file size (in bytes) allowed for asset upload | > 0 | Y | |
| `IPFS_BASIC_AUTH_SECRET` | If using Infura, put auth token here, or leave blank for Kubo RPC | string | N | blank |
| `IPFS_BASIC_AUTH_USER` | If using Infura, put Project ID here, or leave blank for Kubo RPC | string | N | blank |
| `IPFS_ENDPOINT` | URL to IPFS endpoint | URL | Y | |
| `IPFS_GATEWAY_URL` | IPFS gateway URL. '[CID]' is a token that will be replaced with an actual content ID | URL template | Y | |
| `ASSET_EXPIRATION_INTERVAL_SECONDS` | Number of seconds to keep completed asset entrie in the cache before expiring them | > 0 | Y | |
| `ASSET_UPLOAD_VERIFICATION_DELAY_SECONDS` | Base delay in seconds used for exponential backoff while waiting for uploaded assets to be verified available before publishing a content notice | >= 0 | Y | |
| `BATCH_INTERVAL_SECONDS` | Number of seconds between content publishing batches. This is so that the service waits a reasonable amount of time for additional content to publishing before submitting a batch--it represents a trade-off between maximum batch fullness and minimal wait time for published content | > 0 | Y | |
| `BATCH_MAX_COUNT` | Maximum number of items that can be submitted in a single batch | > 0 | Y | |
| `FILE_UPLOAD_MAX_SIZE_IN_BYTES` | Max file size (in bytes) allowed for asset upload | > 0 | Y | |
| `IPFS_BASIC_AUTH_SECRET` | If using Infura, put auth token here, or leave blank for Kubo RPC | string | N | blank |
| `IPFS_BASIC_AUTH_USER` | If using Infura, put Project ID here, or leave blank for Kubo RPC | string | N | blank |
| `IPFS_ENDPOINT` | URL to IPFS endpoint | URL | Y | |
| `IPFS_GATEWAY_URL` | IPFS gateway URL. '[CID]' is a token that will be replaced with an actual content ID | URL template | Y | |
7 changes: 6 additions & 1 deletion services/content-publishing/apps/api/src/api.module.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,12 @@ import { AssetControllerV1, ContentControllerV1, ProfileControllerV1 } from './c
{
imports: [ConfigModule],
useFactory: (configService: ConfigService) => ({
config: [{ url: configService.redisUrl.toString(), keyPrefix: configService.cacheKeyPrefix }],
config: [
{
url: configService.redisUrl.toString(),
keyPrefix: configService.cacheKeyPrefix,
},
],
}),
inject: [ConfigService],
},
Expand Down
10 changes: 8 additions & 2 deletions services/content-publishing/apps/worker/src/request_processor/dsnp.announcement.processor.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,10 @@ export class DsnpAnnouncementProcessor {
attachment: attachments,
});
const noteString = JSON.stringify(note);
const [cid, hash] = await this.pinBufferToIPFS(Buffer.from(noteString));
const toUint8Array = new TextEncoder();
const encoded = toUint8Array.encode(noteString);

const [cid, hash] = await this.pinBufferToIPFS(Buffer.from(encoded));
const ipfsUrl = this.formIpfsUrl(cid);
return [cid, ipfsUrl, hash];
}
Expand Down Expand Up @@ -397,8 +400,11 @@ export class DsnpAnnouncementProcessor {
icon: attachments,
tag: this.prepareTags(content.profile.tag),
};
const toUint8Array = new TextEncoder();
const profileString = JSON.stringify(profileActivity);
const [cid, hash] = await this.pinBufferToIPFS(Buffer.from(profileString));
const profileEncoded = toUint8Array.encode(profileString);

const [cid, hash] = await this.pinBufferToIPFS(Buffer.from(profileEncoded));
return createProfile(dsnpUserId, this.formIpfsUrl(cid), hash);
}

Expand Down
Loading

0 comments on commit fa91f25

Please sign in to comment.