From b46c57be3096bc31f37c627194f4f1f892e00e43 Mon Sep 17 00:00:00 2001 From: Michael Taylor Date: Fri, 12 Apr 2024 14:46:00 -0400 Subject: [PATCH 1/9] feat: check if home profile synced before retirement --- src/api/registry.js | 1697 +++++++++++++++++---------------- src/tasks/sync-retirements.js | 513 +++++----- 2 files changed, 1115 insertions(+), 1095 deletions(-) diff --git a/src/api/registry.js b/src/api/registry.js index 8cc3c11..a62b03f 100644 --- a/src/api/registry.js +++ b/src/api/registry.js @@ -1,842 +1,855 @@ -const superagent = require("superagent"); -const Datalayer = require("chia-datalayer"); -const { CONFIG } = require("../config"); -const { logger } = require("../logger"); -const wallet = require("../chia/wallet"); -const utils = require("../utils"); -const constants = require("../constants"); -const { Mutex } = require("async-mutex"); - -const mutex = new Mutex(); - -const registryUri = utils.generateUriForHostAndPort( - CONFIG().CADT.PROTOCOL, - CONFIG().CADT.HOST, - CONFIG().CADT.PORT -); - -/** - * Appends Registry API Key to the request headers if available. - * - * @param {Object} [headers={}] - Optional headers to extend - * @returns {Object} Headers with API Key added if available - */ -const maybeAppendRegistryApiKey = (headers = {}) => { - if (CONFIG().CADT.API_KEY) { - headers["x-api-key"] = CONFIG().CADT.API_KEY; - } - return headers; -}; - -/** - * Commits staging data to the warehouse. - * - * @returns {Promise} The response body - */ -const commitStagingData = async () => { - try { - logger.debug(`POST ${registryUri}/v1/staging/commit`); - const response = await superagent - .post(`${registryUri}/v1/staging/commit`) - .set(maybeAppendRegistryApiKey()); - - await utils.waitFor(5000); - await wallet.waitForAllTransactionsToConfirm(); - await utils.waitFor(5000); - await waitForRegistryDataSync(); - - return response.body; - } catch (error) { - logger.error(`Could not commit staging data: ${error.message}`); - - // Log additional information if present in the error object - if (error.response && error.response.body) { - logger.error( - `Additional error details: ${JSON.stringify(error.response.body)}` - ); - } - - return null; - } -}; - -/** - * Cleans a unit object before updating it. - * - * @param {Object} unit - The unit to be updated - * @returns {Object} The cleaned unit - */ -const sanitizeUnitForUpdate = (unit) => { - const cleanedUnit = { ...unit }; - - delete cleanedUnit?.issuance?.orgUid; - delete cleanedUnit.issuanceId; - delete cleanedUnit.orgUid; - delete cleanedUnit.serialNumberBlock; - delete cleanedUnit.timeStaged; - - Object.keys(cleanedUnit).forEach((key) => { - if (cleanedUnit[key] === null) { - delete cleanedUnit[key]; - } - }); - - return cleanedUnit; -}; - -/** - * Updates a given unit. - * - * @param {Object} unit - The unit to update - * @returns {Promise} The response body - */ -const updateUnit = async (unit) => { - try { - logger.debug(`PUT ${registryUri}/v1/units`); - const cleanedUnit = sanitizeUnitForUpdate(unit); - const response = await superagent - .put(`${registryUri}/v1/units`) - .send(cleanedUnit) - .set(maybeAppendRegistryApiKey({ "Content-Type": "application/json" })); - - if (response.status === 403) { - throw new Error( - "Registry API key is invalid, please check your config.yaml." - ); - } - - return response?.body; - } catch (error) { - logger.error(`Could not update unit: ${error.message}`); - - // Log additional information if present in the error object - if (error.response && error.response.body) { - logger.error( - `Additional error details: ${JSON.stringify(error.response.body)}` - ); - } - - return null; - } -}; - -/** - * Retires a given unit. - * - * @param {Object} unit - The unit to retire - * @param {string} beneficiaryName - The name of the beneficiary - * @param {string} beneficiaryAddress - The address of the beneficiary - * @returns {Promise} The response body - */ -const retireUnit = async (unit, beneficiaryName, beneficiaryAddress) => { - const cleanedUnit = sanitizeUnitForUpdate(unit); - if (beneficiaryName) { - cleanedUnit.unitOwner = beneficiaryName; - } - if (beneficiaryAddress) { - cleanedUnit.unitStatusReason = beneficiaryAddress; - } - cleanedUnit.unitStatus = "Retired"; - - logger.info(`Retiring whole unit ${unit.warehouseUnitId}`); - return await updateUnit(cleanedUnit); -}; - -/** - * Fetches all pages of asset unit blocks by a marketplace identifier and aggregates the data. - * - * @param {string} marketplaceIdentifier - The marketplace identifier - * @returns {Promise} Aggregate of data from all pages - */ -const getAssetUnitBlocks = async (marketplaceIdentifier) => { - const aggregateData = []; - let currentPage = 1; - let pageCount = 1; - try { - do { - logger.debug(`Fetching page ${currentPage} for marketplaceIdentifier: ${marketplaceIdentifier}`); - const response = await superagent - .get(`${registryUri}/v1/units`) - .query({ - filter: `marketplaceIdentifier:${marketplaceIdentifier}:eq`, - page: currentPage, - limit: 2, - }) - .set(maybeAppendRegistryApiKey()); - - if (response.status === 403) { - throw new Error("Registry API key is invalid, please check your config.yaml."); - } - - aggregateData.push(...response.body.data); - pageCount = response.body.pageCount; - currentPage++; - } while (currentPage <= pageCount); - - return aggregateData; - } catch (error) { - logger.error(`Could not get asset unit blocks from registry: ${error.message}`); - - if (error.response && error.response.body) { - logger.error(`Additional error details: ${JSON.stringify(error.response.body)}`); - } - - return []; - } -}; - - -/** - * Gets the last processed block height. - * - * @returns {Promise} The last processed height or null - */ -const getLastProcessedHeight = async () => { - try { - const homeOrgUid = await getHomeOrgUid(); - logger.debug(`GET ${registryUri}/v1/organizations/metadata`); - const response = await superagent - .get(`${registryUri}/v1/organizations/metadata`) - .query({ orgUid: homeOrgUid }) - .set(maybeAppendRegistryApiKey()); - - if (response.status === 403) { - throw new Error( - "Registry API key is invalid, please check your config.yaml." - ); - } - - return response.status === 200 - ? Number(response.body["lastRetiredBlockHeight"] || 0) - : null; - } catch (error) { - logger.error(`Could not get last processed height: ${error.message}`); - - // Log additional information if present in the error object - if (error.response && error.response.body) { - logger.error( - `Additional error details: ${JSON.stringify(error.response.body)}` - ); - } - - return null; - } -}; - -/** - * Gets the home organization UID. - * - * @returns {Promise} The home organization UID or null - */ -const getHomeOrgUid = async () => { - const homeOrg = await getHomeOrg(); - return homeOrg ? homeOrg.orgUid : null; -}; - -/** - * Gets the home organization. - * - * @returns {Promise} The home organization or null - */ -const getHomeOrg = async () => { - try { - logger.debug(`GET ${registryUri}/v1/organizations`); - const response = await superagent - .get(`${registryUri}/v1/organizations`) - .set(maybeAppendRegistryApiKey()); - - if (response.status === 403) { - throw new Error( - "Registry API key is invalid, please check your config.yaml." - ); - } - - if (response.status !== 200) { - throw new Error(`Received non-200 status code: ${response.status}`); - } - - const orgArray = Object.keys(response.body).map( - (key) => response.body[key] - ); - - const homeOrg = orgArray.find((org) => org.isHome) || null; - - if (homeOrg.orgUid === "PENDING") { - return null; - } - - return homeOrg; - } catch (error) { - logger.error(`Could not get home org: ${error.message}`); - - // Log additional information if present in the error object - if (error.response && error.response.body) { - logger.error( - `Additional error details: ${JSON.stringify(error.response.body)}` - ); - } - - return null; - } -}; - -/** - * Sets the last processed block height. - * - * @param {number} height - The last processed height - * @returns {Promise} The response body - */ -const setLastProcessedHeight = async (height) => { - try { - await wallet.waitForAllTransactionsToConfirm(); - await utils.waitFor(5000); - await waitForRegistryDataSync(); - - logger.debug(`POST ${registryUri}/v1/organizations/metadata`); - const response = await superagent - .post(`${registryUri}/v1/organizations/metadata`) - .send({ lastRetiredBlockHeight: height.toString() }) - .set(maybeAppendRegistryApiKey({ "Content-Type": "application/json" })); - - if (response.status === 403) { - throw new Error( - "Registry API key is invalid, please check your config.yaml." - ); - } - - const data = response.body; - - if ( - response.status !== 200 || - data.message !== - "Home org currently being updated, will be completed soon." - ) { - logger.fatal( - `CRITICAL ERROR: Could not set last processed height in registry.` - ); - return; - } - - await wallet.waitForAllTransactionsToConfirm(); - await utils.waitFor(5000); - await waitForRegistryDataSync(); - - return data; - } catch (error) { - logger.error(`Could not set last processed height: ${error.message}`); - - // Log additional information if present in the error object - if (error.response && error.response.body) { - logger.error( - `Additional error details: ${JSON.stringify(error.response.body)}` - ); - logger.error(`Additional error details: ${JSON.stringify(error)}`); - } - - return null; - } -}; - -/** - * Confirms token registration on the warehouse. - * - * @async - * @function - * @param {number} [retry=0] - The retry count. - * @returns {Promise} Returns a Promise that resolves to true if the token registration is confirmed, or false otherwise. - * @throws {Error} Throws an error if the Registry API key is invalid. - */ -const confirmTokenRegistrationOnWarehouse = async (retry = 0) => { - if (process.env.NODE_ENV === "test") { - return true; - } - - if (retry > 60) return false; - - try { - await utils.waitFor(30000); - - logger.debug(`GET ${registryUri}/v1/staging/hasPendingTransactions`); - const response = await superagent - .get(`${registryUri}/v1/staging/hasPendingTransactions`) - .set(maybeAppendRegistryApiKey()); - - if (response.status === 403) { - throw new Error( - "Registry API key is invalid, please check your config.yaml." - ); - } - - if (response.body?.confirmed) return true; - - await utils.waitFor(30000); - return confirmTokenRegistrationOnWarehouse(retry + 1); - } catch (error) { - logger.error( - `Error confirming token registration on registry: ${error.message}` - ); - - if (error.response?.body) { - logger.error( - `Additional error details: ${JSON.stringify(error.response.body)}` - ); - } - - return false; - } -}; - -/** - * Registers a token creation on the registry. - * - * @param {Object} token - The token to be registered. - * @param {string} warehouseUnitId - The ID of the warehouse unit. - * @returns {Promise} Returns true if successful, null otherwise. - */ -const registerTokenCreationOnRegistry = async (token, warehouseUnitId) => { - try { - await waitForRegistryDataSync(); - - const coreRegistryMode = CONFIG().GENERAL.CORE_REGISTRY_MODE; - const metadataUrl = `${registryUri}/v1/organizations/metadata`; - const apiKeyHeaders = maybeAppendRegistryApiKey({ - "Content-Type": "application/json", - }); - - if (coreRegistryMode) { - token.detokenization = { mod_hash: "", public_key: "", signature: "" }; - } - - logger.debug(`GET ${metadataUrl}`); - const metaDataResponse = await superagent - .get(metadataUrl) - .query({ orgUid: token.org_uid }) - .set(apiKeyHeaders); - - const metaData = metaDataResponse.body; - - if (metaDataResponse.status === 403) { - throw new Error( - "Registry API key is invalid, please check your config.yaml." - ); - } - - if (!metaData[token.asset_id]) { - logger.debug(`POST ${metadataUrl}`); - const response = await superagent - .post(metadataUrl) - .send({ [token.asset_id]: JSON.stringify(token) }) - .set(apiKeyHeaders); - - if (response.status === 403) { - throw new Error( - "Registry API key is invalid, please check your config.yaml." - ); - } - } - - if (coreRegistryMode && (await confirmTokenRegistrationOnWarehouse())) { - await updateUnitMarketplaceIdentifierWithAssetId( - warehouseUnitId, - token.asset_id - ); - } - - return true; - } catch (error) { - logger.error( - `Could not register token creation in registry: ${error.message}` - ); - - if (error.response?.body) { - logger.error( - `Additional error details: ${JSON.stringify(error.response.body)}` - ); - } - - return null; - } -}; - -/** - * Updates the marketplace identifier of a unit with an asset ID. - * - * @async - * @function - * @param {string} warehouseUnitId - The warehouse unit ID to be updated. - * @param {string} asset_id - The new asset ID to be set as marketplace identifier. - * @returns {Promise} Returns a Promise that resolves to the updated unit data if successful, or null if an error occurs. - * @throws {Error} Throws an error if the Registry API key is invalid. - */ -const updateUnitMarketplaceIdentifierWithAssetId = async ( - warehouseUnitId, - asset_id -) => { - try { - logger.debug(`GET ${registryUri}/v1/units`); - const getResponse = await superagent - .get(`${registryUri}/v1/units`) - .query({ warehouseUnitId }) - .set(maybeAppendRegistryApiKey()); - - if (getResponse.status === 403) { - throw new Error( - "Registry API key is invalid, please check your config.yaml." - ); - } - - const unit = { - ...sanitizeUnitForUpdate(getResponse.body), - marketplaceIdentifier: asset_id, - marketplace: "Tokenized on Chia", - }; - - logger.debug(`PUT ${registryUri}/v1/units`); - const putResponse = await superagent - .put(`${registryUri}/v1/units`) - .send(unit) - .set(maybeAppendRegistryApiKey({ "Content-Type": "application/json" })); - - if (putResponse.status === 403) { - throw new Error( - "Registry API key is invalid, please check your config.yaml." - ); - } - - await commitStagingData(); - await utils.waitFor(5000); - await wallet.waitForAllTransactionsToConfirm(); - await utils.waitFor(5000); - await waitForRegistryDataSync(); - - return putResponse?.body; - } catch (error) { - logger.error( - `Could not update unit marketplace identifier with asset id: ${error.message}` - ); - - if (error.response?.body) { - logger.error( - `Additional error details: ${JSON.stringify(error.response.body)}` - ); - } - - return null; - } -}; - -/** - * Fetches metadata for a specific organization. - * - * @param {string} orgUid - The unique identifier for the organization - * @returns {Promise} The organization metadata - */ -const getOrgMetaData = async (orgUid) => { - try { - const url = `${registryUri}/v1/organizations/metadata?orgUid=${orgUid}`; - logger.debug(`GET ${url}`); - const response = await superagent.get(url).set(maybeAppendRegistryApiKey()); - - if (response.status === 403) { - throw new Error( - "Registry API key is invalid, please check your config.yaml." - ); - } - - return response.body; - } catch (error) { - logger.error(`Could not get org metadata: ${error.message}`); - - // Log additional information if present in the error object - if (error.response && error.response.body) { - logger.error( - `Additional error details: ${JSON.stringify(error.response.body)}` - ); - } - - throw new Error(`Could not get org metadata: ${error}`); - } -}; - -/** - * Waits for the registry data to synchronize. - * - * @param {object} [options] - Function options. - * @param {boolean} [options.throwOnEmptyRegistry=false] - Flag to throw error on empty registry. - * @returns {Promise} - */ -const waitForRegistryDataSync = async (options = {}) => { - if (process.env.NODE_ENV === "test") { - return; - } - - await mutex.waitForUnlock(); - - let isFirstSyncAfterFailure = false; - - if (!mutex.isLocked()) { - const releaseMutex = await mutex.acquire(); - try { - const opts = { throwOnEmptyRegistry: false, ...options }; - - if (process.env.NODE_ENV === "test") { - return; - } - - while (true) { - await utils.waitFor(5000); - - const config = CONFIG().CHIA; - const dataLayerConfig = { - datalayer_host: config.DATALAYER_HOST, - wallet_host: config.WALLET_HOST, - certificate_folder_path: config.CERTIFICATE_FOLDER_PATH, - allowUnverifiedCert: config.ALLOW_SELF_SIGNED_CERTIFICATES, - }; - - if (["debug", "trace"].includes(CONFIG().GENERAL.LOG_LEVEL)) { - dataLayerConfig.verbose = true; - } - - const datalayer = new Datalayer(dataLayerConfig); - const homeOrg = await getHomeOrg(); - - if (!homeOrg) { - logger.warn( - "Cannot find the home org from the Registry. Please verify your Registry is running and you have created a Home Organization." - ); - isFirstSyncAfterFailure = true; - continue; - } - - const onChainRegistryRoot = await datalayer.getRoot({ - id: homeOrg.registryId, - }); - - if (!onChainRegistryRoot.confirmed) { - logger.debug("Waiting for Registry root to confirm"); - isFirstSyncAfterFailure = true; - continue; - } - - if ( - onChainRegistryRoot.hash === constants.emptySingletonHash && - opts.throwOnEmptyRegistry - ) { - throw new Error( - "Registry is empty. Please add some data to run auto retirement task." - ); - } - - if (onChainRegistryRoot.hash !== homeOrg.registryHash) { - logger.debug( - `Waiting for Registry to sync with latest registry root. - ${JSON.stringify( - { - onChainRoot: onChainRegistryRoot.hash, - homeOrgRegistryRoot: homeOrg.registryHash, - }, - null, - 2 - )}` - ); - isFirstSyncAfterFailure = true; - continue; - } - - const onChainOrgRoot = await datalayer.getRoot({ id: homeOrg.orgUid }); - - if (!onChainOrgRoot.confirmed) { - logger.debug("Waiting for Organization root to confirm"); - continue; - } - - if (onChainOrgRoot.hash !== homeOrg.orgHash) { - logger.debug( - `Waiting for Registry to sync with latest organization root. , - ${JSON.stringify( - { - onChainRoot: onChainOrgRoot.hash, - homeOrgRoot: homeOrg.orgHash, - }, - null, - 2 - )}` - ); - isFirstSyncAfterFailure = true; - continue; - } - - // Log the message if conditions are met for the first time after failure - if (isFirstSyncAfterFailure) { - logger.info("CADT is SYNCED! Proceeding with the task."); - } - - // Exit the loop if all conditions are met - break; - } - } finally { - releaseMutex(); - } - } -}; - -/** - * Gets the tokenized unit by asset ID. - * - * @param {string} assetId - The unique identifier for the asset - * @returns {Promise} The tokenized unit data - */ -const getTokenizedUnitByAssetId = async (assetId) => { - try { - const url = `${registryUri}/v1/units?marketplaceIdentifiers=${assetId}&page=1&limit=100`; - logger.debug(`GET ${url}`); - const response = await superagent.get(url).set(maybeAppendRegistryApiKey()); - - if (response.status === 403) { - throw new Error( - "Registry API key is invalid, please check your config.yaml." - ); - } - - return response?.body?.data || []; - } catch (error) { - logger.error(`Could not get tokenized unit by asset id: ${error.message}`); - - // Log additional information if present in the error object - if (error.response && error.response.body) { - logger.error( - `Additional error details: ${JSON.stringify(error.response.body)}` - ); - } - - throw new Error(`Could not get tokenized unit by asset id: ${error}`); - } -}; - -/** - * Gets project data by warehouse project ID. - * - * @param {string} warehouseProjectId - The unique identifier for the warehouse project - * @returns {Promise} The project data - */ -const getProjectByWarehouseProjectId = async (warehouseProjectId) => { - try { - const url = `${registryUri}/v1/projects?projectIds=${warehouseProjectId}`; - logger.debug(`GET ${url}`); - const response = await superagent.get(url).set(maybeAppendRegistryApiKey()); - - if (response.status === 403) { - throw new Error( - "Registry API key is invalid, please check your config.yaml." - ); - } - - return response.body[0]; - } catch (error) { - logger.error(`Could not get corresponding project data: ${error.message}`); - - // Log additional information if present in the error object - if (error.response && error.response.body) { - logger.error( - `Additional error details: ${JSON.stringify(error.response.body)}` - ); - } - - throw new Error(`Could not get corresponding project data: ${error}`); - } -}; - -const deleteStagingData = () => { - return superagent.delete(`${registryUri}/v1/staging/clean`); -}; - -const splitUnit = async ({ - unit, - amount, - beneficiaryName, - beneficiaryAddress, -}) => { - logger.info(`Splitting unit ${unit.warehouseUnitId} by ${amount}`); - - // Parse the serialNumberBlock - const { unitBlockStart, unitBlockEnd } = utils.parseSerialNumber( - unit.serialNumberBlock - ); - - if (!unitBlockStart && !unitBlockEnd) { - console.error("serialNumberBlock is not in the correct format"); - return; - } - - const totalUnits = parseInt(unitBlockEnd) - parseInt(unitBlockStart) + 1; - - if (amount >= totalUnits) { - throw new Error("Amount must be less than total units in the block"); - } - - const payload = { - warehouseUnitId: unit.warehouseUnitId, - records: [ - { - unitCount: amount, - marketplace: unit.marketplace, - marketplaceIdentifier: unit.marketplaceIdentifier, - unitStatus: "Retired", - unitOwner: beneficiaryName, - unitStatusReason: beneficiaryAddress, - }, - { - unitCount: totalUnits - amount, - marketplace: unit.marketplace, - marketplaceIdentifier: unit.marketplaceIdentifier, - }, - ], - }; - - try { - logger.debug(`POST ${registryUri}/v1/units/split`); - const response = await superagent - .post(`${registryUri}/v1/units/split`) - .send(JSON.stringify(payload)) - .set(maybeAppendRegistryApiKey({ "Content-Type": "application/json" })); - - if (response.status === 403) { - throw new Error( - "Registry API key is invalid, please check your config.yaml." - ); - } - - return response.body; - } catch (error) { - logger.error(`Could not split unit on registry: ${error.message}`); - - // Log additional information if present in the error object - if (error.response && error.response.body) { - logger.error( - `Additional error details: ${JSON.stringify(error.response.body)}` - ); - } - - return null; - } -}; - -module.exports = { - commitStagingData, - sanitizeUnitForUpdate, - updateUnit, - retireUnit, - getAssetUnitBlocks, - getLastProcessedHeight, - getHomeOrgUid, - getHomeOrg, - setLastProcessedHeight, - registerTokenCreationOnRegistry, - getOrgMetaData, - deleteStagingData, - getTokenizedUnitByAssetId, - getProjectByWarehouseProjectId, - splitUnit, - waitForRegistryDataSync, -}; +const superagent = require("superagent"); +const Datalayer = require("chia-datalayer"); +const { CONFIG } = require("../config"); +const { logger } = require("../logger"); +const wallet = require("../chia/wallet"); +const utils = require("../utils"); +const constants = require("../constants"); +const { Mutex } = require("async-mutex"); + +const mutex = new Mutex(); + +const registryUri = utils.generateUriForHostAndPort( + CONFIG().CADT.PROTOCOL, + CONFIG().CADT.HOST, + CONFIG().CADT.PORT +); + +/** + * Appends Registry API Key to the request headers if available. + * + * @param {Object} [headers={}] - Optional headers to extend + * @returns {Object} Headers with API Key added if available + */ +const maybeAppendRegistryApiKey = (headers = {}) => { + if (CONFIG().CADT.API_KEY) { + headers["x-api-key"] = CONFIG().CADT.API_KEY; + } + return headers; +}; + +/** + * Commits staging data to the warehouse. + * + * @returns {Promise} The response body + */ +const commitStagingData = async () => { + try { + logger.debug(`POST ${registryUri}/v1/staging/commit`); + const response = await superagent + .post(`${registryUri}/v1/staging/commit`) + .set(maybeAppendRegistryApiKey()); + + await utils.waitFor(5000); + await wallet.waitForAllTransactionsToConfirm(); + await utils.waitFor(5000); + await waitForRegistryDataSync(); + + return response.body; + } catch (error) { + logger.error(`Could not commit staging data: ${error.message}`); + + // Log additional information if present in the error object + if (error.response && error.response.body) { + logger.error( + `Additional error details: ${JSON.stringify(error.response.body)}` + ); + } + + return null; + } +}; + +/** + * Cleans a unit object before updating it. + * + * @param {Object} unit - The unit to be updated + * @returns {Object} The cleaned unit + */ +const sanitizeUnitForUpdate = (unit) => { + const cleanedUnit = { ...unit }; + + delete cleanedUnit?.issuance?.orgUid; + delete cleanedUnit.issuanceId; + delete cleanedUnit.orgUid; + delete cleanedUnit.serialNumberBlock; + delete cleanedUnit.timeStaged; + + Object.keys(cleanedUnit).forEach((key) => { + if (cleanedUnit[key] === null) { + delete cleanedUnit[key]; + } + }); + + return cleanedUnit; +}; + +/** + * Updates a given unit. + * + * @param {Object} unit - The unit to update + * @returns {Promise} The response body + */ +const updateUnit = async (unit) => { + try { + logger.debug(`PUT ${registryUri}/v1/units`); + const cleanedUnit = sanitizeUnitForUpdate(unit); + const response = await superagent + .put(`${registryUri}/v1/units`) + .send(cleanedUnit) + .set(maybeAppendRegistryApiKey({ "Content-Type": "application/json" })); + + if (response.status === 403) { + throw new Error( + "Registry API key is invalid, please check your config.yaml." + ); + } + + return response?.body; + } catch (error) { + logger.error(`Could not update unit: ${error.message}`); + + // Log additional information if present in the error object + if (error.response && error.response.body) { + logger.error( + `Additional error details: ${JSON.stringify(error.response.body)}` + ); + } + + return null; + } +}; + +/** + * Retires a given unit. + * + * @param {Object} unit - The unit to retire + * @param {string} beneficiaryName - The name of the beneficiary + * @param {string} beneficiaryAddress - The address of the beneficiary + * @returns {Promise} The response body + */ +const retireUnit = async (unit, beneficiaryName, beneficiaryAddress) => { + const cleanedUnit = sanitizeUnitForUpdate(unit); + if (beneficiaryName) { + cleanedUnit.unitOwner = beneficiaryName; + } + if (beneficiaryAddress) { + cleanedUnit.unitStatusReason = beneficiaryAddress; + } + cleanedUnit.unitStatus = "Retired"; + + logger.info(`Retiring whole unit ${unit.warehouseUnitId}`); + return await updateUnit(cleanedUnit); +}; + +/** + * Fetches all pages of asset unit blocks by a marketplace identifier and aggregates the data. + * + * @param {string} marketplaceIdentifier - The marketplace identifier + * @returns {Promise} Aggregate of data from all pages + */ +const getAssetUnitBlocks = async (marketplaceIdentifier) => { + const aggregateData = []; + let currentPage = 1; + let pageCount = 1; + try { + do { + logger.debug(`Fetching page ${currentPage} for marketplaceIdentifier: ${marketplaceIdentifier}`); + const response = await superagent + .get(`${registryUri}/v1/units`) + .query({ + filter: `marketplaceIdentifier:${marketplaceIdentifier}:eq`, + page: currentPage, + limit: 2, + }) + .set(maybeAppendRegistryApiKey()); + + if (response.status === 403) { + throw new Error("Registry API key is invalid, please check your config.yaml."); + } + + aggregateData.push(...response.body.data); + pageCount = response.body.pageCount; + currentPage++; + } while (currentPage <= pageCount); + + return aggregateData; + } catch (error) { + logger.error(`Could not get asset unit blocks from registry: ${error.message}`); + + if (error.response && error.response.body) { + logger.error(`Additional error details: ${JSON.stringify(error.response.body)}`); + } + + return []; + } +}; + + +const getHomeOrgSyncStatus = async () => { + logger.debug(`GET ${registryUri}/v1/organizations/sync_status`); + const response = await superagent.get(`${registryUri}/v1/organizations/sync_status`).set(maybeAppendRegistryApiKey()); + + if (response.status === 403) { + throw new Error("Registry API key is invalid, please check your config.yaml."); + } + + return response.body; +} + + +/** + * Gets the last processed block height. + * + * @returns {Promise} The last processed height or null + */ +const getLastProcessedHeight = async () => { + try { + const homeOrgUid = await getHomeOrgUid(); + logger.debug(`GET ${registryUri}/v1/organizations/metadata`); + const response = await superagent + .get(`${registryUri}/v1/organizations/metadata`) + .query({ orgUid: homeOrgUid }) + .set(maybeAppendRegistryApiKey()); + + if (response.status === 403) { + throw new Error( + "Registry API key is invalid, please check your config.yaml." + ); + } + + return response.status === 200 + ? Number(response.body["lastRetiredBlockHeight"] || 0) + : null; + } catch (error) { + logger.error(`Could not get last processed height: ${error.message}`); + + // Log additional information if present in the error object + if (error.response && error.response.body) { + logger.error( + `Additional error details: ${JSON.stringify(error.response.body)}` + ); + } + + return null; + } +}; + +/** + * Gets the home organization UID. + * + * @returns {Promise} The home organization UID or null + */ +const getHomeOrgUid = async () => { + const homeOrg = await getHomeOrg(); + return homeOrg ? homeOrg.orgUid : null; +}; + +/** + * Gets the home organization. + * + * @returns {Promise} The home organization or null + */ +const getHomeOrg = async () => { + try { + logger.debug(`GET ${registryUri}/v1/organizations`); + const response = await superagent + .get(`${registryUri}/v1/organizations`) + .set(maybeAppendRegistryApiKey()); + + if (response.status === 403) { + throw new Error( + "Registry API key is invalid, please check your config.yaml." + ); + } + + if (response.status !== 200) { + throw new Error(`Received non-200 status code: ${response.status}`); + } + + const orgArray = Object.keys(response.body).map( + (key) => response.body[key] + ); + + const homeOrg = orgArray.find((org) => org.isHome) || null; + + if (homeOrg.orgUid === "PENDING") { + return null; + } + + return homeOrg; + } catch (error) { + logger.error(`Could not get home org: ${error.message}`); + + // Log additional information if present in the error object + if (error.response && error.response.body) { + logger.error( + `Additional error details: ${JSON.stringify(error.response.body)}` + ); + } + + return null; + } +}; + +/** + * Sets the last processed block height. + * + * @param {number} height - The last processed height + * @returns {Promise} The response body + */ +const setLastProcessedHeight = async (height) => { + try { + await wallet.waitForAllTransactionsToConfirm(); + await utils.waitFor(5000); + await waitForRegistryDataSync(); + + logger.debug(`POST ${registryUri}/v1/organizations/metadata`); + const response = await superagent + .post(`${registryUri}/v1/organizations/metadata`) + .send({ lastRetiredBlockHeight: height.toString() }) + .set(maybeAppendRegistryApiKey({ "Content-Type": "application/json" })); + + if (response.status === 403) { + throw new Error( + "Registry API key is invalid, please check your config.yaml." + ); + } + + const data = response.body; + + if ( + response.status !== 200 || + data.message !== + "Home org currently being updated, will be completed soon." + ) { + logger.fatal( + `CRITICAL ERROR: Could not set last processed height in registry.` + ); + return; + } + + await wallet.waitForAllTransactionsToConfirm(); + await utils.waitFor(5000); + await waitForRegistryDataSync(); + + return data; + } catch (error) { + logger.error(`Could not set last processed height: ${error.message}`); + + // Log additional information if present in the error object + if (error.response && error.response.body) { + logger.error( + `Additional error details: ${JSON.stringify(error.response.body)}` + ); + logger.error(`Additional error details: ${JSON.stringify(error)}`); + } + + return null; + } +}; + +/** + * Confirms token registration on the warehouse. + * + * @async + * @function + * @param {number} [retry=0] - The retry count. + * @returns {Promise} Returns a Promise that resolves to true if the token registration is confirmed, or false otherwise. + * @throws {Error} Throws an error if the Registry API key is invalid. + */ +const confirmTokenRegistrationOnWarehouse = async (retry = 0) => { + if (process.env.NODE_ENV === "test") { + return true; + } + + if (retry > 60) return false; + + try { + await utils.waitFor(30000); + + logger.debug(`GET ${registryUri}/v1/staging/hasPendingTransactions`); + const response = await superagent + .get(`${registryUri}/v1/staging/hasPendingTransactions`) + .set(maybeAppendRegistryApiKey()); + + if (response.status === 403) { + throw new Error( + "Registry API key is invalid, please check your config.yaml." + ); + } + + if (response.body?.confirmed) return true; + + await utils.waitFor(30000); + return confirmTokenRegistrationOnWarehouse(retry + 1); + } catch (error) { + logger.error( + `Error confirming token registration on registry: ${error.message}` + ); + + if (error.response?.body) { + logger.error( + `Additional error details: ${JSON.stringify(error.response.body)}` + ); + } + + return false; + } +}; + +/** + * Registers a token creation on the registry. + * + * @param {Object} token - The token to be registered. + * @param {string} warehouseUnitId - The ID of the warehouse unit. + * @returns {Promise} Returns true if successful, null otherwise. + */ +const registerTokenCreationOnRegistry = async (token, warehouseUnitId) => { + try { + await waitForRegistryDataSync(); + + const coreRegistryMode = CONFIG().GENERAL.CORE_REGISTRY_MODE; + const metadataUrl = `${registryUri}/v1/organizations/metadata`; + const apiKeyHeaders = maybeAppendRegistryApiKey({ + "Content-Type": "application/json", + }); + + if (coreRegistryMode) { + token.detokenization = { mod_hash: "", public_key: "", signature: "" }; + } + + logger.debug(`GET ${metadataUrl}`); + const metaDataResponse = await superagent + .get(metadataUrl) + .query({ orgUid: token.org_uid }) + .set(apiKeyHeaders); + + const metaData = metaDataResponse.body; + + if (metaDataResponse.status === 403) { + throw new Error( + "Registry API key is invalid, please check your config.yaml." + ); + } + + if (!metaData[token.asset_id]) { + logger.debug(`POST ${metadataUrl}`); + const response = await superagent + .post(metadataUrl) + .send({ [token.asset_id]: JSON.stringify(token) }) + .set(apiKeyHeaders); + + if (response.status === 403) { + throw new Error( + "Registry API key is invalid, please check your config.yaml." + ); + } + } + + if (coreRegistryMode && (await confirmTokenRegistrationOnWarehouse())) { + await updateUnitMarketplaceIdentifierWithAssetId( + warehouseUnitId, + token.asset_id + ); + } + + return true; + } catch (error) { + logger.error( + `Could not register token creation in registry: ${error.message}` + ); + + if (error.response?.body) { + logger.error( + `Additional error details: ${JSON.stringify(error.response.body)}` + ); + } + + return null; + } +}; + +/** + * Updates the marketplace identifier of a unit with an asset ID. + * + * @async + * @function + * @param {string} warehouseUnitId - The warehouse unit ID to be updated. + * @param {string} asset_id - The new asset ID to be set as marketplace identifier. + * @returns {Promise} Returns a Promise that resolves to the updated unit data if successful, or null if an error occurs. + * @throws {Error} Throws an error if the Registry API key is invalid. + */ +const updateUnitMarketplaceIdentifierWithAssetId = async ( + warehouseUnitId, + asset_id +) => { + try { + logger.debug(`GET ${registryUri}/v1/units`); + const getResponse = await superagent + .get(`${registryUri}/v1/units`) + .query({ warehouseUnitId }) + .set(maybeAppendRegistryApiKey()); + + if (getResponse.status === 403) { + throw new Error( + "Registry API key is invalid, please check your config.yaml." + ); + } + + const unit = { + ...sanitizeUnitForUpdate(getResponse.body), + marketplaceIdentifier: asset_id, + marketplace: "Tokenized on Chia", + }; + + logger.debug(`PUT ${registryUri}/v1/units`); + const putResponse = await superagent + .put(`${registryUri}/v1/units`) + .send(unit) + .set(maybeAppendRegistryApiKey({ "Content-Type": "application/json" })); + + if (putResponse.status === 403) { + throw new Error( + "Registry API key is invalid, please check your config.yaml." + ); + } + + await commitStagingData(); + await utils.waitFor(5000); + await wallet.waitForAllTransactionsToConfirm(); + await utils.waitFor(5000); + await waitForRegistryDataSync(); + + return putResponse?.body; + } catch (error) { + logger.error( + `Could not update unit marketplace identifier with asset id: ${error.message}` + ); + + if (error.response?.body) { + logger.error( + `Additional error details: ${JSON.stringify(error.response.body)}` + ); + } + + return null; + } +}; + +/** + * Fetches metadata for a specific organization. + * + * @param {string} orgUid - The unique identifier for the organization + * @returns {Promise} The organization metadata + */ +const getOrgMetaData = async (orgUid) => { + try { + const url = `${registryUri}/v1/organizations/metadata?orgUid=${orgUid}`; + logger.debug(`GET ${url}`); + const response = await superagent.get(url).set(maybeAppendRegistryApiKey()); + + if (response.status === 403) { + throw new Error( + "Registry API key is invalid, please check your config.yaml." + ); + } + + return response.body; + } catch (error) { + logger.error(`Could not get org metadata: ${error.message}`); + + // Log additional information if present in the error object + if (error.response && error.response.body) { + logger.error( + `Additional error details: ${JSON.stringify(error.response.body)}` + ); + } + + throw new Error(`Could not get org metadata: ${error}`); + } +}; + +/** + * Waits for the registry data to synchronize. + * + * @param {object} [options] - Function options. + * @param {boolean} [options.throwOnEmptyRegistry=false] - Flag to throw error on empty registry. + * @returns {Promise} + */ +const waitForRegistryDataSync = async (options = {}) => { + if (process.env.NODE_ENV === "test") { + return; + } + + await mutex.waitForUnlock(); + + let isFirstSyncAfterFailure = false; + + if (!mutex.isLocked()) { + const releaseMutex = await mutex.acquire(); + try { + const opts = { throwOnEmptyRegistry: false, ...options }; + + if (process.env.NODE_ENV === "test") { + return; + } + + while (true) { + await utils.waitFor(5000); + + const config = CONFIG().CHIA; + const dataLayerConfig = { + datalayer_host: config.DATALAYER_HOST, + wallet_host: config.WALLET_HOST, + certificate_folder_path: config.CERTIFICATE_FOLDER_PATH, + allowUnverifiedCert: config.ALLOW_SELF_SIGNED_CERTIFICATES, + }; + + if (["debug", "trace"].includes(CONFIG().GENERAL.LOG_LEVEL)) { + dataLayerConfig.verbose = true; + } + + const datalayer = new Datalayer(dataLayerConfig); + const homeOrg = await getHomeOrg(); + + if (!homeOrg) { + logger.warn( + "Cannot find the home org from the Registry. Please verify your Registry is running and you have created a Home Organization." + ); + isFirstSyncAfterFailure = true; + continue; + } + + const onChainRegistryRoot = await datalayer.getRoot({ + id: homeOrg.registryId, + }); + + if (!onChainRegistryRoot.confirmed) { + logger.debug("Waiting for Registry root to confirm"); + isFirstSyncAfterFailure = true; + continue; + } + + if ( + onChainRegistryRoot.hash === constants.emptySingletonHash && + opts.throwOnEmptyRegistry + ) { + throw new Error( + "Registry is empty. Please add some data to run auto retirement task." + ); + } + + if (onChainRegistryRoot.hash !== homeOrg.registryHash) { + logger.debug( + `Waiting for Registry to sync with latest registry root. + ${JSON.stringify( + { + onChainRoot: onChainRegistryRoot.hash, + homeOrgRegistryRoot: homeOrg.registryHash, + }, + null, + 2 + )}` + ); + isFirstSyncAfterFailure = true; + continue; + } + + const onChainOrgRoot = await datalayer.getRoot({ id: homeOrg.orgUid }); + + if (!onChainOrgRoot.confirmed) { + logger.debug("Waiting for Organization root to confirm"); + continue; + } + + if (onChainOrgRoot.hash !== homeOrg.orgHash) { + logger.debug( + `Waiting for Registry to sync with latest organization root. , + ${JSON.stringify( + { + onChainRoot: onChainOrgRoot.hash, + homeOrgRoot: homeOrg.orgHash, + }, + null, + 2 + )}` + ); + isFirstSyncAfterFailure = true; + continue; + } + + // Log the message if conditions are met for the first time after failure + if (isFirstSyncAfterFailure) { + logger.info("CADT is SYNCED! Proceeding with the task."); + } + + // Exit the loop if all conditions are met + break; + } + } finally { + releaseMutex(); + } + } +}; + +/** + * Gets the tokenized unit by asset ID. + * + * @param {string} assetId - The unique identifier for the asset + * @returns {Promise} The tokenized unit data + */ +const getTokenizedUnitByAssetId = async (assetId) => { + try { + const url = `${registryUri}/v1/units?marketplaceIdentifiers=${assetId}&page=1&limit=100`; + logger.debug(`GET ${url}`); + const response = await superagent.get(url).set(maybeAppendRegistryApiKey()); + + if (response.status === 403) { + throw new Error( + "Registry API key is invalid, please check your config.yaml." + ); + } + + return response?.body?.data || []; + } catch (error) { + logger.error(`Could not get tokenized unit by asset id: ${error.message}`); + + // Log additional information if present in the error object + if (error.response && error.response.body) { + logger.error( + `Additional error details: ${JSON.stringify(error.response.body)}` + ); + } + + throw new Error(`Could not get tokenized unit by asset id: ${error}`); + } +}; + +/** + * Gets project data by warehouse project ID. + * + * @param {string} warehouseProjectId - The unique identifier for the warehouse project + * @returns {Promise} The project data + */ +const getProjectByWarehouseProjectId = async (warehouseProjectId) => { + try { + const url = `${registryUri}/v1/projects?projectIds=${warehouseProjectId}`; + logger.debug(`GET ${url}`); + const response = await superagent.get(url).set(maybeAppendRegistryApiKey()); + + if (response.status === 403) { + throw new Error( + "Registry API key is invalid, please check your config.yaml." + ); + } + + return response.body[0]; + } catch (error) { + logger.error(`Could not get corresponding project data: ${error.message}`); + + // Log additional information if present in the error object + if (error.response && error.response.body) { + logger.error( + `Additional error details: ${JSON.stringify(error.response.body)}` + ); + } + + throw new Error(`Could not get corresponding project data: ${error}`); + } +}; + +const deleteStagingData = () => { + return superagent.delete(`${registryUri}/v1/staging/clean`); +}; + +const splitUnit = async ({ + unit, + amount, + beneficiaryName, + beneficiaryAddress, +}) => { + logger.info(`Splitting unit ${unit.warehouseUnitId} by ${amount}`); + + // Parse the serialNumberBlock + const { unitBlockStart, unitBlockEnd } = utils.parseSerialNumber( + unit.serialNumberBlock + ); + + if (!unitBlockStart && !unitBlockEnd) { + console.error("serialNumberBlock is not in the correct format"); + return; + } + + const totalUnits = parseInt(unitBlockEnd) - parseInt(unitBlockStart) + 1; + + if (amount >= totalUnits) { + throw new Error("Amount must be less than total units in the block"); + } + + const payload = { + warehouseUnitId: unit.warehouseUnitId, + records: [ + { + unitCount: amount, + marketplace: unit.marketplace, + marketplaceIdentifier: unit.marketplaceIdentifier, + unitStatus: "Retired", + unitOwner: beneficiaryName, + unitStatusReason: beneficiaryAddress, + }, + { + unitCount: totalUnits - amount, + marketplace: unit.marketplace, + marketplaceIdentifier: unit.marketplaceIdentifier, + }, + ], + }; + + try { + logger.debug(`POST ${registryUri}/v1/units/split`); + const response = await superagent + .post(`${registryUri}/v1/units/split`) + .send(JSON.stringify(payload)) + .set(maybeAppendRegistryApiKey({ "Content-Type": "application/json" })); + + if (response.status === 403) { + throw new Error( + "Registry API key is invalid, please check your config.yaml." + ); + } + + return response.body; + } catch (error) { + logger.error(`Could not split unit on registry: ${error.message}`); + + // Log additional information if present in the error object + if (error.response && error.response.body) { + logger.error( + `Additional error details: ${JSON.stringify(error.response.body)}` + ); + } + + return null; + } +}; + +module.exports = { + commitStagingData, + sanitizeUnitForUpdate, + updateUnit, + retireUnit, + getAssetUnitBlocks, + getLastProcessedHeight, + getHomeOrgUid, + getHomeOrg, + setLastProcessedHeight, + registerTokenCreationOnRegistry, + getOrgMetaData, + deleteStagingData, + getTokenizedUnitByAssetId, + getProjectByWarehouseProjectId, + splitUnit, + waitForRegistryDataSync, + getHomeOrgSyncStatus +}; diff --git a/src/tasks/sync-retirements.js b/src/tasks/sync-retirements.js index 8b92e38..e3fdba6 100644 --- a/src/tasks/sync-retirements.js +++ b/src/tasks/sync-retirements.js @@ -1,253 +1,260 @@ -const { SimpleIntervalJob, Task } = require("toad-scheduler"); - -const wallet = require("../chia/wallet"); -const registry = require("../api/registry"); -const retirementExplorer = require("../api/retirement-explorer"); -const { logger } = require("../logger"); -const { CONFIG } = require("../config"); - -let isTaskInProgress = false; - -/** - * Scheduler task definition. - */ -const task = new Task("sync-retirements", async () => { - try { - if (!isTaskInProgress) { - logger.task("Starting sync-retirements task"); - isTaskInProgress = true; - await startSyncRetirementsTask(); - } - } catch (error) { - logger.error(`Error in sync-retirements task: ${error.message}`); - } finally { - isTaskInProgress = false; - } -}); - -/** - * Job configuration and initiation. - */ -const job = new SimpleIntervalJob( - { - seconds: - CONFIG().TOKENIZATION_ENGINE.TASKS - .SYNC_RETIREMENTS_TO_REGISTRY_INTERVAL_SECONDS, - runImmediately: true, - }, - task, - // @ts-ignore - "sync-retirements" -); - -/** - * Starts the sync-retirements task, which retrieves and processes retirement activities. - * @returns {Promise} - */ -const startSyncRetirementsTask = async () => { - try { - await registry.waitForRegistryDataSync({ throwOnEmptyRegistry: true }); - const homeOrg = await registry.getHomeOrg(); - - if (!homeOrg) { - logger.warn( - "Can not attain home organization from the registry, skipping sync-retirements task" - ); - return; - } - - const lastProcessedHeight = await registry.getLastProcessedHeight(); - if (lastProcessedHeight == null) { - logger.warn( - "Can not attain the last Processed Retirement Height from the registry, skipping sync-retirements task" - ); - return; - } - - await getAndProcessActivities(homeOrg, lastProcessedHeight); - } catch (error) { - logger.error(`Error in sync-retirements task: ${error.message}`); - } -}; - -/** - * Get and process retirement activities from the API. - * @param {number} minHeight - Minimum block height to start. - * @returns {Promise} - */ -const getAndProcessActivities = async (homeOrg, minHeight = 0) => { - try { - let page = 1; - const limit = 10; - while (true) { - const retirements = await retirementExplorer.getRetirementActivities( - page, - limit, - minHeight - ); - - logger.debug(`Retirement activities: ${JSON.stringify(retirements)}`); - - if (!retirements?.length) { - break; - } - const ownedRetirements = retirements.filter( - (activity) => activity?.token?.org_uid === homeOrg.orgUid - ); - - if (!ownedRetirements?.length) { - page++; - continue; - } - - logger.debug( - `Owned Retirement activities: ${JSON.stringify(retirements)}` - ); - - for (const activity of ownedRetirements) { - // You can only autoretire your own units - logger.info(`PROCESSING RETIREMENT ACTIVITY: ${activity.coin_id}`); - await processResult({ - marketplaceIdentifier: activity.cw_unit.marketplaceIdentifier, - amount: activity.amount / 1000, - beneficiaryName: activity.beneficiary_name, - beneficiaryAddress: activity.beneficiary_address, - }); - } - - const highestHeight = calcHighestActivityHeight(retirements); - - // Only set the latest processed height if we actually processed something - // This prevents us from setting the last processed height to the same height - // if we don't have any units to retire and prevents an unneeded transaction - if (highestHeight >= minHeight) { - await registry.setLastProcessedHeight(highestHeight); - } - - page++; - } - } catch (error) { - throw new Error(`Cannot get retirement activities: ${error.message}`); - } -}; - -/** - * Process retirement result. - * @param {Object} params - Parameters. - * @param {string} params.marketplaceIdentifier - Marketplace Identifier. - * @param {number} params.amount - Amount to retire. - * @param {string} params.beneficiaryName - Beneficiary's name. - * @param {string} params.beneficiaryAddress - Beneficiary's address. - * @returns {Promise} - */ -const processResult = async ({ - marketplaceIdentifier, - amount, - beneficiaryName, - beneficiaryAddress, -}) => { - try { - await registry.waitForRegistryDataSync(); - const unitBlocks = await registry.getAssetUnitBlocks(marketplaceIdentifier); - - const units = unitBlocks - .filter((unit) => unit.unitStatus !== "Retired") - .sort((a, b) => b.unitCount - a.unitCount); - - if (!units || units.length === 0) { - logger.task(`No units for ${marketplaceIdentifier}`); - return; - } - - const remainingAmountToRetire = await processUnits( - units, - amount, - beneficiaryName, - beneficiaryAddress - ); - - if (remainingAmountToRetire > 0) { - await registry.deleteStagingData(); - throw new Error("Total unitCount lower than needed retire amount."); - } - - await registry.commitStagingData(); - logger.task("Auto Retirement Process Complete"); - } catch (err) { - throw new Error("Could not retire unit block"); - } -}; - -/** - * Process individual units for retirement. - * @param {Array} units - Array of unit blocks to be processed. - * @param {number} amount - Amount to retire. - * @param {string} beneficiaryName - Beneficiary's name. - * @param {string} beneficiaryAddress - Beneficiary's address. - * @returns {Promise} - */ -const processUnits = async ( - units, - amount, - beneficiaryName, - beneficiaryAddress -) => { - let remainingAmountToRetire = amount; - for (const unit of units) { - if (remainingAmountToRetire <= 0) { - break; - } - const { unitCount } = unit; - - if (isNaN(unitCount)) { - logger.error( - `unitCount for unit ${unit.warehouseUnitId} is not a number. Skipping this unit.` - ); - break; - } else { - logger.task( - `Retiring ${unitCount} units for ${unit.warehouseUnitId} with ${remainingAmountToRetire} remaining` - ); - } - - if (unitCount <= remainingAmountToRetire) { - await registry.retireUnit(unit, beneficiaryName, beneficiaryAddress); - remainingAmountToRetire -= unitCount; - } else { - await registry.splitUnit({ - unit, - amount: remainingAmountToRetire, - beneficiaryName, - beneficiaryAddress, - }); - remainingAmountToRetire = 0; - } - await wallet.waitForAllTransactionsToConfirm(); - } - - return remainingAmountToRetire; -}; - -/** - * Helper function to find the highest height among retirement activities. - * @param {Array} activities - Array of retirement activities. - * @returns {number} The highest block height. - */ -const calcHighestActivityHeight = (activities) => { - let highestHeight = 0; - - activities.forEach((activity) => { - const height = activity.height; - - if (height > highestHeight) { - highestHeight = height; - } - }); - - return highestHeight; -}; - -module.exports = { - startSyncRetirementsTask, - job, -}; +const { SimpleIntervalJob, Task } = require("toad-scheduler"); + +const wallet = require("../chia/wallet"); +const registry = require("../api/registry"); +const retirementExplorer = require("../api/retirement-explorer"); +const { logger } = require("../logger"); +const { CONFIG } = require("../config"); + +let isTaskInProgress = false; + +/** + * Scheduler task definition. + */ +const task = new Task("sync-retirements", async () => { + try { + if (!isTaskInProgress) { + logger.task("Starting sync-retirements task"); + isTaskInProgress = true; + await startSyncRetirementsTask(); + } + } catch (error) { + logger.error(`Error in sync-retirements task: ${error.message}`); + } finally { + isTaskInProgress = false; + } +}); + +/** + * Job configuration and initiation. + */ +const job = new SimpleIntervalJob( + { + seconds: + CONFIG().TOKENIZATION_ENGINE.TASKS + .SYNC_RETIREMENTS_TO_REGISTRY_INTERVAL_SECONDS, + runImmediately: true, + }, + task, + // @ts-ignore + "sync-retirements" +); + +/** + * Starts the sync-retirements task, which retrieves and processes retirement activities. + * @returns {Promise} + */ +const startSyncRetirementsTask = async () => { + try { + await registry.waitForRegistryDataSync({ throwOnEmptyRegistry: true }); + const homeOrg = await registry.getHomeOrg(); + + if (!homeOrg) { + logger.warn( + "Can not attain home organization from the registry, skipping sync-retirements task" + ); + return; + } + + const syncStatus = await registry.getHomeOrgSyncStatus(); + if (!syncStatus?.home_org_profile_synced) { + logger.warn( + "Home organization sync is not complete, skipping sync-retirements task" + ); + } + + const lastProcessedHeight = await registry.getLastProcessedHeight(); + if (lastProcessedHeight == null) { + logger.warn( + "Can not attain the last Processed Retirement Height from the registry, skipping sync-retirements task" + ); + return; + } + + await getAndProcessActivities(homeOrg, lastProcessedHeight); + } catch (error) { + logger.error(`Error in sync-retirements task: ${error.message}`); + } +}; + +/** + * Get and process retirement activities from the API. + * @param {number} minHeight - Minimum block height to start. + * @returns {Promise} + */ +const getAndProcessActivities = async (homeOrg, minHeight = 0) => { + try { + let page = 1; + const limit = 10; + while (true) { + const retirements = await retirementExplorer.getRetirementActivities( + page, + limit, + minHeight + ); + + logger.debug(`Retirement activities: ${JSON.stringify(retirements)}`); + + if (!retirements?.length) { + break; + } + const ownedRetirements = retirements.filter( + (activity) => activity?.token?.org_uid === homeOrg.orgUid + ); + + if (!ownedRetirements?.length) { + page++; + continue; + } + + logger.debug( + `Owned Retirement activities: ${JSON.stringify(retirements)}` + ); + + for (const activity of ownedRetirements) { + // You can only autoretire your own units + logger.info(`PROCESSING RETIREMENT ACTIVITY: ${activity.coin_id}`); + await processResult({ + marketplaceIdentifier: activity.cw_unit.marketplaceIdentifier, + amount: activity.amount / 1000, + beneficiaryName: activity.beneficiary_name, + beneficiaryAddress: activity.beneficiary_address, + }); + } + + const highestHeight = calcHighestActivityHeight(retirements); + + // Only set the latest processed height if we actually processed something + // This prevents us from setting the last processed height to the same height + // if we don't have any units to retire and prevents an unneeded transaction + if (highestHeight >= minHeight) { + await registry.setLastProcessedHeight(highestHeight); + } + + page++; + } + } catch (error) { + throw new Error(`Cannot get retirement activities: ${error.message}`); + } +}; + +/** + * Process retirement result. + * @param {Object} params - Parameters. + * @param {string} params.marketplaceIdentifier - Marketplace Identifier. + * @param {number} params.amount - Amount to retire. + * @param {string} params.beneficiaryName - Beneficiary's name. + * @param {string} params.beneficiaryAddress - Beneficiary's address. + * @returns {Promise} + */ +const processResult = async ({ + marketplaceIdentifier, + amount, + beneficiaryName, + beneficiaryAddress, +}) => { + try { + await registry.waitForRegistryDataSync(); + const unitBlocks = await registry.getAssetUnitBlocks(marketplaceIdentifier); + + const units = unitBlocks + .filter((unit) => unit.unitStatus !== "Retired") + .sort((a, b) => b.unitCount - a.unitCount); + + if (!units || units.length === 0) { + logger.task(`No units for ${marketplaceIdentifier}`); + return; + } + + const remainingAmountToRetire = await processUnits( + units, + amount, + beneficiaryName, + beneficiaryAddress + ); + + if (remainingAmountToRetire > 0) { + await registry.deleteStagingData(); + throw new Error("Total unitCount lower than needed retire amount."); + } + + await registry.commitStagingData(); + logger.task("Auto Retirement Process Complete"); + } catch (err) { + throw new Error("Could not retire unit block"); + } +}; + +/** + * Process individual units for retirement. + * @param {Array} units - Array of unit blocks to be processed. + * @param {number} amount - Amount to retire. + * @param {string} beneficiaryName - Beneficiary's name. + * @param {string} beneficiaryAddress - Beneficiary's address. + * @returns {Promise} + */ +const processUnits = async ( + units, + amount, + beneficiaryName, + beneficiaryAddress +) => { + let remainingAmountToRetire = amount; + for (const unit of units) { + if (remainingAmountToRetire <= 0) { + break; + } + const { unitCount } = unit; + + if (isNaN(unitCount)) { + logger.error( + `unitCount for unit ${unit.warehouseUnitId} is not a number. Skipping this unit.` + ); + break; + } else { + logger.task( + `Retiring ${unitCount} units for ${unit.warehouseUnitId} with ${remainingAmountToRetire} remaining` + ); + } + + if (unitCount <= remainingAmountToRetire) { + await registry.retireUnit(unit, beneficiaryName, beneficiaryAddress); + remainingAmountToRetire -= unitCount; + } else { + await registry.splitUnit({ + unit, + amount: remainingAmountToRetire, + beneficiaryName, + beneficiaryAddress, + }); + remainingAmountToRetire = 0; + } + await wallet.waitForAllTransactionsToConfirm(); + } + + return remainingAmountToRetire; +}; + +/** + * Helper function to find the highest height among retirement activities. + * @param {Array} activities - Array of retirement activities. + * @returns {number} The highest block height. + */ +const calcHighestActivityHeight = (activities) => { + let highestHeight = 0; + + activities.forEach((activity) => { + const height = activity.height; + + if (height > highestHeight) { + highestHeight = height; + } + }); + + return highestHeight; +}; + +module.exports = { + startSyncRetirementsTask, + job, +}; From 8f59a1f9b390cee8f77cc24d29cf2ff20fac54ad Mon Sep 17 00:00:00 2001 From: Michael Taylor Date: Fri, 12 Apr 2024 14:52:09 -0400 Subject: [PATCH 2/9] test: fix test --- tests/tasks/sync-retirements.test.js | 766 ++++++++++++++------------- 1 file changed, 384 insertions(+), 382 deletions(-) diff --git a/tests/tasks/sync-retirements.test.js b/tests/tasks/sync-retirements.test.js index 32bddbc..21905bc 100644 --- a/tests/tasks/sync-retirements.test.js +++ b/tests/tasks/sync-retirements.test.js @@ -1,382 +1,384 @@ -const { expect } = require("chai"); -const sinon = require("sinon"); -const nock = require("nock"); - -const syncRetirements = require("../../src/tasks/sync-retirements"); -const registry = require("../../src/api/registry"); -const { logger } = require("../../src/logger"); -const wallet = require("../../src/chia/wallet"); -const retirementExplorer = require("../../src/api/retirement-explorer"); -const ActivityResponseMock = require("../data/ActivityResponseMock"); -const HomeOrgMock = require("../data/HomeOrgMock"); -const OrganizationsMock = require("../data/OrganizationsMock"); -const { CONFIG } = require("../../src/config"); -const { generateUriForHostAndPort } = require("../../src/utils"); - -const registryUri = generateUriForHostAndPort( - CONFIG().CADT.PROTOCOL, - CONFIG().CADT.HOST, - CONFIG().CADT.PORT -); - - -describe("Task: Sync Retirements", () => { - let retirementExplorerGetRetirementActivitiesStub; - let registrySetLastProcessedHeightStub; - let registryRetireUnitStub; - let registrySplitUnitStub; - let registryCommitStagingDataStub; - let registryGetHomeOrgStub; - - beforeEach(() => { - nock(registryUri).get("/v1/organizations").reply(200, OrganizationsMock); - - retirementExplorerGetRetirementActivitiesStub = sinon - .stub(retirementExplorer, "getRetirementActivities") - .resolves([]); - - registrySetLastProcessedHeightStub = sinon - .stub(registry, "setLastProcessedHeight") - .resolves({}); - - registryRetireUnitStub = sinon.stub(registry, "retireUnit").resolves(); - registrySplitUnitStub = sinon.stub(registry, "splitUnit").resolves(); - registryCommitStagingDataStub = sinon - .stub(registry, "commitStagingData") - .resolves(); - - registryGetHomeOrgStub = sinon - .stub(registry, "getHomeOrg") - .resolves(HomeOrgMock); - - sinon.stub(wallet, "waitForAllTransactionsToConfirm").resolves(); - sinon.stub(registry, "waitForRegistryDataSync").resolves(); - }); - - afterEach(async () => { - sinon.restore(); - await new Promise((resolve) => setTimeout(() => resolve(), 1000)); - }); - - it("skips retirement task if no homeorg can be attained by the registry", async () => { - registryGetHomeOrgStub.resolves(null); - - const warnSpy = sinon.spy(logger, "warn"); - - await syncRetirements.startSyncRetirementsTask(); - - expect(warnSpy.called).to.be.true; - expect( - warnSpy.calledWith( - "Can not attain home organization from the registry, skipping sync-retirements task" - ) - ).to.be.true; - - expect(retirementExplorerGetRetirementActivitiesStub.called).to.be.false; - expect(registrySetLastProcessedHeightStub.called).to.be.false; - }); - - it("skips retirement task if the last processed retirement height can not be attained by the registry", async () => { - sinon.stub(registry, "getLastProcessedHeight").resolves(null); - - const warnSpy = sinon.spy(logger, "warn"); - - await syncRetirements.startSyncRetirementsTask(); - - expect(warnSpy.called).to.be.true; - expect( - warnSpy.calledWith( - "Can not attain the last Processed Retirement Height from the registry, skipping sync-retirements task" - ) - ).to.be.true; - - expect(retirementExplorerGetRetirementActivitiesStub.called).to.be.false; - expect(registrySetLastProcessedHeightStub.called).to.be.false; - }); - - it("starts processing activities if valid homeorg and lastProcessedHeight can be attained by the registry", async () => { - // Stub registry methods - - sinon.stub(registry, "getLastProcessedHeight").resolves(12345); - - // Spy on logger.warn method - const warnSpy = sinon.spy(logger, "warn"); - - // Run the method under test - await syncRetirements.startSyncRetirementsTask(); - - // Perform assertions - expect(warnSpy.called).to.be.false; - expect(retirementExplorerGetRetirementActivitiesStub.called).to.be.true; - - // expecting this to be false because the stub is not returning any activities - expect(registrySetLastProcessedHeightStub.called).to.be.false; - }); - - it("skips processing activities that are not from your home org", async () => { - const modifiedHomeOrg = Object.assign({}, HomeOrgMock); - modifiedHomeOrg.orgUid = "DIFFERENT_ORG_UID"; - registryGetHomeOrgStub.resolves(modifiedHomeOrg); - - // Stub registry methods - let registryGetAssetUnitBlocksStub = sinon.stub( - registry, - "getAssetUnitBlocks" - ); - sinon.stub(registry, "getLastProcessedHeight").resolves(12345); - retirementExplorerGetRetirementActivitiesStub - .onFirstCall() - .resolves(ActivityResponseMock.activities); - retirementExplorerGetRetirementActivitiesStub.onSecondCall().resolves([]); - - // Spy on logger.warn method - const warnSpy = sinon.spy(logger, "warn"); - - // Run the method under test - await syncRetirements.startSyncRetirementsTask(); - - // Perform assertions - expect(warnSpy.called).to.be.false; - expect(retirementExplorerGetRetirementActivitiesStub.called).to.be.true; - - expect(registryGetAssetUnitBlocksStub.called).to.be.false; - - // expecting this to be false because we didnt get any activities from our home org - expect(registrySetLastProcessedHeightStub.called).to.be.false; - }); - - it("Does not run the task if the task is already running", () => { - /* implement this test by doing the following - * - call the startSyncRetirementsTask function - * - create a spy for getAndProcessActivities - * - assert that the isTaskInProgress variable is true - * - assert that the getAndProcessActivities function was called - * - call the startSyncRetirementsTask function again - * - assert that the isTaskInProgress variable is still true - * - assert that the getAndProcessActivities function not called a second time - * - restore the stubs - * - restore the spy - */ - }); - - it("does not set last processed height if there are no units to retire", async () => { - sinon.stub(registry, "getLastProcessedHeight").resolves(12345); - - retirementExplorerGetRetirementActivitiesStub.restore(); - retirementExplorerGetRetirementActivitiesStub = sinon.stub( - retirementExplorer, - "getRetirementActivities" - ); - - retirementExplorerGetRetirementActivitiesStub.onFirstCall().resolves([ - { - amount: 5000, - beneficiary_name: "TEST_BENEFICIARY_NAME", - beneficiary_address: "TEST_BENEFICIARY_ADDRESS", - height: 99999, - mode: "PERMISSIONLESS_RETIREMENT", - cw_unit: { - marketplaceIdentifier: "TEST_MARKETPLACE_IDENTIFIER", - }, - token: { - org_uid: HomeOrgMock.orgUid, - }, - }, - { - amount: 5000, - beneficiary_name: "TEST_BENEFICIARY_NAME", - beneficiary_address: "TEST_BENEFICIARY_ADDRESS", - height: 12346, - mode: "PERMISSIONLESS_RETIREMENT", - cw_unit: { - marketplaceIdentifier: "TEST_MARKETPLACE_IDENTIFIER", - }, - token: { - org_uid: HomeOrgMock.orgUid, - }, - }, - ]); - - retirementExplorerGetRetirementActivitiesStub.onSecondCall().resolves([]); - - sinon.stub(registry, "getAssetUnitBlocks").resolves([ - { - unitStatus: "Retired", - unitCount: 10, - marketplaceIdentifier: "TEST_MARKETPLACE_IDENTIFIER", - }, - ]); - - const taskSpy = sinon.spy(logger, "task"); - - // Run the method under test - await syncRetirements.startSyncRetirementsTask(); - - expect(taskSpy.calledWith("No units for TEST_MARKETPLACE_IDENTIFIER")).to.be - .true; - - expect(registryRetireUnitStub.called).to.be.false; - expect(registrySplitUnitStub.called).to.be.false; - - expect(registrySetLastProcessedHeightStub.called).to.be.true; - - // make sure just the highest block is recorded - expect(registrySetLastProcessedHeightStub.args[0][0]).to.equal(99999); - }); - - it("writes the lastProcessedHeight to the registry when the retirments have been processed", () => {}); - - it("sets the post body fields correctly when requesting retirement activities", async () => { - const lastProcessedHeightMock = 12345; - - sinon - .stub(registry, "getLastProcessedHeight") - .resolves(lastProcessedHeightMock); - - await syncRetirements.startSyncRetirementsTask(); - - expect(retirementExplorerGetRetirementActivitiesStub.args[0][2]).to.equal( - lastProcessedHeightMock - ); - - // expecting this to be false because the stub is not returning any activities - expect(registrySetLastProcessedHeightStub.called).to.be.false; - }); - - it("Does not process activities that are marked as already retired", () => {}); - - it("Retires all units when amount is greater than the amount of units available for the unit block", async () => { - sinon.stub(registry, "getLastProcessedHeight").resolves(12345); - - retirementExplorerGetRetirementActivitiesStub.restore(); - retirementExplorerGetRetirementActivitiesStub = sinon.stub( - retirementExplorer, - "getRetirementActivities" - ); - - retirementExplorerGetRetirementActivitiesStub.onFirstCall().resolves([ - { - amount: 10000, - beneficiary_name: "TEST_BENEFICIARY_NAME", - beneficiary_address: "TEST_BENEFICIARY_ADDRESS", - mode: "PERMISSIONLESS_RETIREMENT", - height: 99999, - cw_unit: { - marketplaceIdentifier: "TEST_MARKETPLACE_IDENTIFIER", - }, - token: { - org_uid: HomeOrgMock.orgUid, - }, - }, - ]); - - retirementExplorerGetRetirementActivitiesStub.onSecondCall().resolves([]); - - const unretiredUnit = { - unitStatus: "Held", - unitCount: 10, - marketplaceIdentifier: "TEST_MARKETPLACE_IDENTIFIER", - }; - - sinon.stub(registry, "getAssetUnitBlocks").resolves([unretiredUnit]); - - await syncRetirements.startSyncRetirementsTask(); - - expect(registryRetireUnitStub.calledOnce).to.be.true; - expect(registryRetireUnitStub.args[0][0]).to.equal(unretiredUnit); - expect(registryRetireUnitStub.args[0][1]).to.equal("TEST_BENEFICIARY_NAME"); - expect(registryRetireUnitStub.args[0][2]).to.equal( - "TEST_BENEFICIARY_ADDRESS" - ); - - // make sure just the highest block is recorded - expect(registrySetLastProcessedHeightStub.args[0][0]).to.equal(99999); - }); - - it("Splits the unit block when the amount is less than the amount of units available for the unit block", async () => { - sinon.stub(registry, "getLastProcessedHeight").resolves(12345); - - retirementExplorerGetRetirementActivitiesStub.restore(); - retirementExplorerGetRetirementActivitiesStub = sinon.stub( - retirementExplorer, - "getRetirementActivities" - ); - - retirementExplorerGetRetirementActivitiesStub.onFirstCall().resolves([ - { - amount: 5000, - beneficiary_name: "TEST_BENEFICIARY_NAME", - beneficiary_address: "TEST_BENEFICIARY_ADDRESS", - height: 99999, - mode: "PERMISSIONLESS_RETIREMENT", - cw_unit: { - marketplaceIdentifier: "TEST_MARKETPLACE_IDENTIFIER", - }, - token: { - org_uid: HomeOrgMock.orgUid, - }, - }, - { - amount: 5000, - beneficiary_name: "TEST_BENEFICIARY_NAME", - beneficiary_address: "TEST_BENEFICIARY_ADDRESS", - height: 12344, - mode: "PERMISSIONLESS_RETIREMENT", - cw_unit: { - marketplaceIdentifier: "TEST_MARKETPLACE_IDENTIFIER", - }, - token: { - org_uid: HomeOrgMock.orgUid, - }, - }, - ]); - - retirementExplorerGetRetirementActivitiesStub.onSecondCall().resolves([]); - - const unretiredUnit = { - unitStatus: "Held", - unitCount: 10, - marketplaceIdentifier: "TEST_MARKETPLACE_IDENTIFIER", - }; - - const unretiredUnitAfterSplit = { - unitStatus: "Held", - unitCount: 5, - marketplaceIdentifier: "TEST_MARKETPLACE_IDENTIFIER", - }; - - const registryGetAssetUnitBlocksStub = sinon.stub( - registry, - "getAssetUnitBlocks" - ); - registryGetAssetUnitBlocksStub.onFirstCall().resolves([unretiredUnit]); - registryGetAssetUnitBlocksStub - .onSecondCall() - .resolves([unretiredUnitAfterSplit]); - - await syncRetirements.startSyncRetirementsTask(); - - expect(registrySplitUnitStub.calledOnce).to.be.true; - expect(registrySplitUnitStub.args[0][0].unit).to.equal(unretiredUnit); - expect(registrySplitUnitStub.args[0][0].amount).to.equal(5); - expect(registrySplitUnitStub.args[0][0].beneficiaryName).to.equal( - "TEST_BENEFICIARY_NAME" - ); - expect(registrySplitUnitStub.args[0][0].beneficiaryAddress).to.equal( - "TEST_BENEFICIARY_ADDRESS" - ); - - expect(registryRetireUnitStub.calledOnce).to.be.true; - - expect(registryRetireUnitStub.args[0][0]).to.equal(unretiredUnitAfterSplit); - expect(registryRetireUnitStub.args[0][1]).to.equal("TEST_BENEFICIARY_NAME"); - expect(registryRetireUnitStub.args[0][2]).to.equal( - "TEST_BENEFICIARY_ADDRESS" - ); - - expect(registryCommitStagingDataStub.callCount).to.equal(2); - - // make sure just the highest block is recorded - expect(registrySetLastProcessedHeightStub.args[0][0]).to.equal(99999); - }); -}); +const { expect } = require("chai"); +const sinon = require("sinon"); +const nock = require("nock"); + +const syncRetirements = require("../../src/tasks/sync-retirements"); +const registry = require("../../src/api/registry"); +const { logger } = require("../../src/logger"); +const wallet = require("../../src/chia/wallet"); +const retirementExplorer = require("../../src/api/retirement-explorer"); +const ActivityResponseMock = require("../data/ActivityResponseMock"); +const HomeOrgMock = require("../data/HomeOrgMock"); +const OrganizationsMock = require("../data/OrganizationsMock"); +const { CONFIG } = require("../../src/config"); +const { generateUriForHostAndPort } = require("../../src/utils"); + +const registryUri = generateUriForHostAndPort( + CONFIG().CADT.PROTOCOL, + CONFIG().CADT.HOST, + CONFIG().CADT.PORT +); + + +describe("Task: Sync Retirements", () => { + let retirementExplorerGetRetirementActivitiesStub; + let registrySetLastProcessedHeightStub; + let registryRetireUnitStub; + let registrySplitUnitStub; + let registryCommitStagingDataStub; + let registryGetHomeOrgStub; + let registryGetHomeOrgSyncStatusStub; + + beforeEach(() => { + nock(registryUri).get("/v1/organizations").reply(200, OrganizationsMock); + + retirementExplorerGetRetirementActivitiesStub = sinon + .stub(retirementExplorer, "getRetirementActivities") + .resolves([]); + + registrySetLastProcessedHeightStub = sinon + .stub(registry, "setLastProcessedHeight") + .resolves({}); + + registryGetHomeOrgSyncStatusStub = sinon.stub(registry, "getHomeOrgSyncStatus").resolves({ home_org_profile_synced: true }); + registryRetireUnitStub = sinon.stub(registry, "retireUnit").resolves(); + registrySplitUnitStub = sinon.stub(registry, "splitUnit").resolves(); + registryCommitStagingDataStub = sinon + .stub(registry, "commitStagingData") + .resolves(); + + registryGetHomeOrgStub = sinon + .stub(registry, "getHomeOrg") + .resolves(HomeOrgMock); + + sinon.stub(wallet, "waitForAllTransactionsToConfirm").resolves(); + sinon.stub(registry, "waitForRegistryDataSync").resolves(); + }); + + afterEach(async () => { + sinon.restore(); + await new Promise((resolve) => setTimeout(() => resolve(), 1000)); + }); + + it("skips retirement task if no homeorg can be attained by the registry", async () => { + registryGetHomeOrgStub.resolves(null); + + const warnSpy = sinon.spy(logger, "warn"); + + await syncRetirements.startSyncRetirementsTask(); + + expect(warnSpy.called).to.be.true; + expect( + warnSpy.calledWith( + "Can not attain home organization from the registry, skipping sync-retirements task" + ) + ).to.be.true; + + expect(retirementExplorerGetRetirementActivitiesStub.called).to.be.false; + expect(registrySetLastProcessedHeightStub.called).to.be.false; + }); + + it("skips retirement task if the last processed retirement height can not be attained by the registry", async () => { + sinon.stub(registry, "getLastProcessedHeight").resolves(null); + + const warnSpy = sinon.spy(logger, "warn"); + + await syncRetirements.startSyncRetirementsTask(); + + expect(warnSpy.called).to.be.true; + expect( + warnSpy.calledWith( + "Can not attain the last Processed Retirement Height from the registry, skipping sync-retirements task" + ) + ).to.be.true; + + expect(retirementExplorerGetRetirementActivitiesStub.called).to.be.false; + expect(registrySetLastProcessedHeightStub.called).to.be.false; + }); + + it("starts processing activities if valid homeorg and lastProcessedHeight can be attained by the registry", async () => { + // Stub registry methods + + sinon.stub(registry, "getLastProcessedHeight").resolves(12345); + + // Spy on logger.warn method + const warnSpy = sinon.spy(logger, "warn"); + + // Run the method under test + await syncRetirements.startSyncRetirementsTask(); + + // Perform assertions + expect(warnSpy.called).to.be.false; + expect(retirementExplorerGetRetirementActivitiesStub.called).to.be.true; + + // expecting this to be false because the stub is not returning any activities + expect(registrySetLastProcessedHeightStub.called).to.be.false; + }); + + it("skips processing activities that are not from your home org", async () => { + const modifiedHomeOrg = Object.assign({}, HomeOrgMock); + modifiedHomeOrg.orgUid = "DIFFERENT_ORG_UID"; + registryGetHomeOrgStub.resolves(modifiedHomeOrg); + + // Stub registry methods + let registryGetAssetUnitBlocksStub = sinon.stub( + registry, + "getAssetUnitBlocks" + ); + sinon.stub(registry, "getLastProcessedHeight").resolves(12345); + retirementExplorerGetRetirementActivitiesStub + .onFirstCall() + .resolves(ActivityResponseMock.activities); + retirementExplorerGetRetirementActivitiesStub.onSecondCall().resolves([]); + + // Spy on logger.warn method + const warnSpy = sinon.spy(logger, "warn"); + + // Run the method under test + await syncRetirements.startSyncRetirementsTask(); + + // Perform assertions + expect(warnSpy.called).to.be.false; + expect(retirementExplorerGetRetirementActivitiesStub.called).to.be.true; + + expect(registryGetAssetUnitBlocksStub.called).to.be.false; + + // expecting this to be false because we didnt get any activities from our home org + expect(registrySetLastProcessedHeightStub.called).to.be.false; + }); + + it("Does not run the task if the task is already running", () => { + /* implement this test by doing the following + * - call the startSyncRetirementsTask function + * - create a spy for getAndProcessActivities + * - assert that the isTaskInProgress variable is true + * - assert that the getAndProcessActivities function was called + * - call the startSyncRetirementsTask function again + * - assert that the isTaskInProgress variable is still true + * - assert that the getAndProcessActivities function not called a second time + * - restore the stubs + * - restore the spy + */ + }); + + it("does not set last processed height if there are no units to retire", async () => { + sinon.stub(registry, "getLastProcessedHeight").resolves(12345); + + retirementExplorerGetRetirementActivitiesStub.restore(); + retirementExplorerGetRetirementActivitiesStub = sinon.stub( + retirementExplorer, + "getRetirementActivities" + ); + + retirementExplorerGetRetirementActivitiesStub.onFirstCall().resolves([ + { + amount: 5000, + beneficiary_name: "TEST_BENEFICIARY_NAME", + beneficiary_address: "TEST_BENEFICIARY_ADDRESS", + height: 99999, + mode: "PERMISSIONLESS_RETIREMENT", + cw_unit: { + marketplaceIdentifier: "TEST_MARKETPLACE_IDENTIFIER", + }, + token: { + org_uid: HomeOrgMock.orgUid, + }, + }, + { + amount: 5000, + beneficiary_name: "TEST_BENEFICIARY_NAME", + beneficiary_address: "TEST_BENEFICIARY_ADDRESS", + height: 12346, + mode: "PERMISSIONLESS_RETIREMENT", + cw_unit: { + marketplaceIdentifier: "TEST_MARKETPLACE_IDENTIFIER", + }, + token: { + org_uid: HomeOrgMock.orgUid, + }, + }, + ]); + + retirementExplorerGetRetirementActivitiesStub.onSecondCall().resolves([]); + + sinon.stub(registry, "getAssetUnitBlocks").resolves([ + { + unitStatus: "Retired", + unitCount: 10, + marketplaceIdentifier: "TEST_MARKETPLACE_IDENTIFIER", + }, + ]); + + const taskSpy = sinon.spy(logger, "task"); + + // Run the method under test + await syncRetirements.startSyncRetirementsTask(); + + expect(taskSpy.calledWith("No units for TEST_MARKETPLACE_IDENTIFIER")).to.be + .true; + + expect(registryRetireUnitStub.called).to.be.false; + expect(registrySplitUnitStub.called).to.be.false; + + expect(registrySetLastProcessedHeightStub.called).to.be.true; + + // make sure just the highest block is recorded + expect(registrySetLastProcessedHeightStub.args[0][0]).to.equal(99999); + }); + + it("writes the lastProcessedHeight to the registry when the retirments have been processed", () => {}); + + it("sets the post body fields correctly when requesting retirement activities", async () => { + const lastProcessedHeightMock = 12345; + + sinon + .stub(registry, "getLastProcessedHeight") + .resolves(lastProcessedHeightMock); + + await syncRetirements.startSyncRetirementsTask(); + + expect(retirementExplorerGetRetirementActivitiesStub.args[0][2]).to.equal( + lastProcessedHeightMock + ); + + // expecting this to be false because the stub is not returning any activities + expect(registrySetLastProcessedHeightStub.called).to.be.false; + }); + + it("Does not process activities that are marked as already retired", () => {}); + + it("Retires all units when amount is greater than the amount of units available for the unit block", async () => { + sinon.stub(registry, "getLastProcessedHeight").resolves(12345); + + retirementExplorerGetRetirementActivitiesStub.restore(); + retirementExplorerGetRetirementActivitiesStub = sinon.stub( + retirementExplorer, + "getRetirementActivities" + ); + + retirementExplorerGetRetirementActivitiesStub.onFirstCall().resolves([ + { + amount: 10000, + beneficiary_name: "TEST_BENEFICIARY_NAME", + beneficiary_address: "TEST_BENEFICIARY_ADDRESS", + mode: "PERMISSIONLESS_RETIREMENT", + height: 99999, + cw_unit: { + marketplaceIdentifier: "TEST_MARKETPLACE_IDENTIFIER", + }, + token: { + org_uid: HomeOrgMock.orgUid, + }, + }, + ]); + + retirementExplorerGetRetirementActivitiesStub.onSecondCall().resolves([]); + + const unretiredUnit = { + unitStatus: "Held", + unitCount: 10, + marketplaceIdentifier: "TEST_MARKETPLACE_IDENTIFIER", + }; + + sinon.stub(registry, "getAssetUnitBlocks").resolves([unretiredUnit]); + + await syncRetirements.startSyncRetirementsTask(); + + expect(registryRetireUnitStub.calledOnce).to.be.true; + expect(registryRetireUnitStub.args[0][0]).to.equal(unretiredUnit); + expect(registryRetireUnitStub.args[0][1]).to.equal("TEST_BENEFICIARY_NAME"); + expect(registryRetireUnitStub.args[0][2]).to.equal( + "TEST_BENEFICIARY_ADDRESS" + ); + + // make sure just the highest block is recorded + expect(registrySetLastProcessedHeightStub.args[0][0]).to.equal(99999); + }); + + it("Splits the unit block when the amount is less than the amount of units available for the unit block", async () => { + sinon.stub(registry, "getLastProcessedHeight").resolves(12345); + + retirementExplorerGetRetirementActivitiesStub.restore(); + retirementExplorerGetRetirementActivitiesStub = sinon.stub( + retirementExplorer, + "getRetirementActivities" + ); + + retirementExplorerGetRetirementActivitiesStub.onFirstCall().resolves([ + { + amount: 5000, + beneficiary_name: "TEST_BENEFICIARY_NAME", + beneficiary_address: "TEST_BENEFICIARY_ADDRESS", + height: 99999, + mode: "PERMISSIONLESS_RETIREMENT", + cw_unit: { + marketplaceIdentifier: "TEST_MARKETPLACE_IDENTIFIER", + }, + token: { + org_uid: HomeOrgMock.orgUid, + }, + }, + { + amount: 5000, + beneficiary_name: "TEST_BENEFICIARY_NAME", + beneficiary_address: "TEST_BENEFICIARY_ADDRESS", + height: 12344, + mode: "PERMISSIONLESS_RETIREMENT", + cw_unit: { + marketplaceIdentifier: "TEST_MARKETPLACE_IDENTIFIER", + }, + token: { + org_uid: HomeOrgMock.orgUid, + }, + }, + ]); + + retirementExplorerGetRetirementActivitiesStub.onSecondCall().resolves([]); + + const unretiredUnit = { + unitStatus: "Held", + unitCount: 10, + marketplaceIdentifier: "TEST_MARKETPLACE_IDENTIFIER", + }; + + const unretiredUnitAfterSplit = { + unitStatus: "Held", + unitCount: 5, + marketplaceIdentifier: "TEST_MARKETPLACE_IDENTIFIER", + }; + + const registryGetAssetUnitBlocksStub = sinon.stub( + registry, + "getAssetUnitBlocks" + ); + registryGetAssetUnitBlocksStub.onFirstCall().resolves([unretiredUnit]); + registryGetAssetUnitBlocksStub + .onSecondCall() + .resolves([unretiredUnitAfterSplit]); + + await syncRetirements.startSyncRetirementsTask(); + + expect(registrySplitUnitStub.calledOnce).to.be.true; + expect(registrySplitUnitStub.args[0][0].unit).to.equal(unretiredUnit); + expect(registrySplitUnitStub.args[0][0].amount).to.equal(5); + expect(registrySplitUnitStub.args[0][0].beneficiaryName).to.equal( + "TEST_BENEFICIARY_NAME" + ); + expect(registrySplitUnitStub.args[0][0].beneficiaryAddress).to.equal( + "TEST_BENEFICIARY_ADDRESS" + ); + + expect(registryRetireUnitStub.calledOnce).to.be.true; + + expect(registryRetireUnitStub.args[0][0]).to.equal(unretiredUnitAfterSplit); + expect(registryRetireUnitStub.args[0][1]).to.equal("TEST_BENEFICIARY_NAME"); + expect(registryRetireUnitStub.args[0][2]).to.equal( + "TEST_BENEFICIARY_ADDRESS" + ); + + expect(registryCommitStagingDataStub.callCount).to.equal(2); + + // make sure just the highest block is recorded + expect(registrySetLastProcessedHeightStub.args[0][0]).to.equal(99999); + }); +}); From 0571f989330b22157d335042ba5c40bd90532ab7 Mon Sep 17 00:00:00 2001 From: William Wills Date: Tue, 30 Jul 2024 12:45:42 -0400 Subject: [PATCH 3/9] feat: bump to node v20 --- .nvmrc | 1 + package-lock.json | 15 ++++++++++++--- package.json | 3 +++ 3 files changed, 16 insertions(+), 3 deletions(-) create mode 100644 .nvmrc diff --git a/.nvmrc b/.nvmrc new file mode 100644 index 0000000..9bdb657 --- /dev/null +++ b/.nvmrc @@ -0,0 +1 @@ +20.16 \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 0e369d8..ebb291b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "climate-tokenization-engine", - "version": "1.3.20", + "version": "1.3.21", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "climate-tokenization-engine", - "version": "1.3.20", + "version": "1.3.21", "license": "Apache-2.0", "dependencies": { "@chia-carbon/core-registry-config": "^1.0.4", @@ -42,6 +42,9 @@ "proxyquire": "^2.1.3", "sinon": "^11.1.2", "sinon-chai": "^3.7.0" + }, + "engines": { + "node": ">=20.16" } }, "node_modules/@ampproject/remapping": { @@ -2509,6 +2512,7 @@ "version": "16.4.5", "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.5.tgz", "integrity": "sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==", + "license": "BSD-2-Clause", "engines": { "node": ">=12" }, @@ -2875,6 +2879,7 @@ "resolved": "https://registry.npmjs.org/fill-keys/-/fill-keys-1.0.2.tgz", "integrity": "sha512-tcgI872xXjwFF4xgQmLxi76GnwJG3g/3isB1l4/G5Z4zrbddGpBjqZCO9oEAcB5wX0Hj/5iQB3toxfO7in1hHA==", "dev": true, + "license": "MIT", "dependencies": { "is-object": "~1.0.1", "merge-descriptors": "~1.0.0" @@ -3623,6 +3628,7 @@ "resolved": "https://registry.npmjs.org/is-object/-/is-object-1.0.2.tgz", "integrity": "sha512-2rRIahhZr2UWb45fIOuvZGpFtz0TyOZLf32KxBbSoUCeZR495zCKlWUKKUByk3geS2eAs7ZAABt0Y/Rx0GiQGA==", "dev": true, + "license": "MIT", "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -4827,7 +4833,8 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/module-not-found-error/-/module-not-found-error-1.0.1.tgz", "integrity": "sha512-pEk4ECWQXV6z2zjhRZUongnLJNUeGQJ3w6OQ5ctGwD+i5o93qjRQUk2Rt6VdNeu3sEP0AB4LcfvdebpxBRVr4g==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/moment": { "version": "2.30.1", @@ -5611,6 +5618,7 @@ "resolved": "https://registry.npmjs.org/proxyquire/-/proxyquire-2.1.3.tgz", "integrity": "sha512-BQWfCqYM+QINd+yawJz23tbBM40VIGXOdDw3X344KcclI/gtBbdWF6SlQ4nK/bYhF9d27KYug9WzljHC6B9Ysg==", "dev": true, + "license": "MIT", "dependencies": { "fill-keys": "^1.0.2", "module-not-found-error": "^1.0.1", @@ -6128,6 +6136,7 @@ "resolved": "https://registry.npmjs.org/sinon-chai/-/sinon-chai-3.7.0.tgz", "integrity": "sha512-mf5NURdUaSdnatJx3uhoBOrY9dtL19fiOtAdT1Azxg3+lNJFiuN0uzaU3xX1LeAfL17kHQhTAJgpsfhbMJMY2g==", "dev": true, + "license": "(BSD-2-Clause OR WTFPL)", "peerDependencies": { "chai": "^4.0.0", "sinon": ">=4.0.0" diff --git a/package.json b/package.json index eed2926..3858d05 100644 --- a/package.json +++ b/package.json @@ -4,6 +4,9 @@ "bin": "./src/server.js", "description": "", "main": "proxy.js", + "engines": { + "node": ">=20.16" + }, "scripts": { "test": "jest --forceExit", "test-ci": "jest --ci", From c67490f956857e4a1595321af09eaed44fe8f6a1 Mon Sep 17 00:00:00 2001 From: William Wills Date: Wed, 7 Aug 2024 16:43:59 -0400 Subject: [PATCH 4/9] feat: app packaged as single application on node20 --- package-lock.json | 257 +++++++++++++++++++++++----------------------- package.json | 10 +- 2 files changed, 136 insertions(+), 131 deletions(-) diff --git a/package-lock.json b/package-lock.json index ebb291b..a60db77 100644 --- a/package-lock.json +++ b/package-lock.json @@ -34,11 +34,11 @@ "climate-tokenization-engine": "src/server.js" }, "devDependencies": { + "@yao-pkg/pkg": "^5.12.0", "chai": "^4.4.1", "cors": "^2.8.5", "jest": "^27.5.1", "nock": "^13.5.3", - "pkg": "^5.8.1", "proxyquire": "^2.1.3", "sinon": "^11.1.2", "sinon-chai": "^3.7.0" @@ -1324,6 +1324,136 @@ "integrity": "sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==", "dev": true }, + "node_modules/@yao-pkg/pkg": { + "version": "5.12.0", + "resolved": "https://registry.npmjs.org/@yao-pkg/pkg/-/pkg-5.12.0.tgz", + "integrity": "sha512-KZVpiDKRi2gtrVtKwhz/ZUKBOicVNggxaYQzPBjULuOLJ/UypTmAz5a2g+utLMn+WogbLE3vLfmC+TWp8v3+aQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/generator": "7.23.0", + "@babel/parser": "7.23.0", + "@babel/types": "7.23.0", + "@yao-pkg/pkg-fetch": "3.5.9", + "chalk": "^4.1.2", + "fs-extra": "^9.1.0", + "globby": "^11.1.0", + "into-stream": "^6.0.0", + "is-core-module": "2.9.0", + "minimatch": "9.0.4", + "minimist": "^1.2.6", + "multistream": "^4.1.0", + "prebuild-install": "7.1.1", + "resolve": "^1.22.0", + "stream-meter": "^1.0.4" + }, + "bin": { + "pkg": "lib-es5/bin.js" + } + }, + "node_modules/@yao-pkg/pkg-fetch": { + "version": "3.5.9", + "resolved": "https://registry.npmjs.org/@yao-pkg/pkg-fetch/-/pkg-fetch-3.5.9.tgz", + "integrity": "sha512-usMwwqFCd2B7k+V87u6kiTesyDSlw+3LpiuYBWe+UgryvSOk/NXjx3XVCub8hQoi0bCREbdQ6NDBqminyHJJrg==", + "dev": true, + "license": "MIT", + "dependencies": { + "chalk": "^4.1.2", + "fs-extra": "^9.1.0", + "https-proxy-agent": "^5.0.0", + "node-fetch": "^2.6.6", + "progress": "^2.0.3", + "semver": "^7.3.5", + "tar-fs": "^2.1.1", + "yargs": "^16.2.0" + }, + "bin": { + "pkg-fetch": "lib-es5/bin.js" + } + }, + "node_modules/@yao-pkg/pkg-fetch/node_modules/semver": { + "version": "7.6.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", + "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/@yao-pkg/pkg/node_modules/@babel/generator": { + "version": "7.23.0", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.23.0.tgz", + "integrity": "sha512-lN85QRR+5IbYrMWM6Y4pE/noaQtg4pNiqeNGX60eqOfo6gtEj6uw/JagelB8vVztSd7R6M5n1+PQkDbHbBRU4g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.23.0", + "@jridgewell/gen-mapping": "^0.3.2", + "@jridgewell/trace-mapping": "^0.3.17", + "jsesc": "^2.5.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@yao-pkg/pkg/node_modules/@babel/parser": { + "version": "7.23.0", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.23.0.tgz", + "integrity": "sha512-vvPKKdMemU85V9WE/l5wZEmImpCtLqbnTvqDS2U1fJ96KrxoW7KrXhNsNCblQlg8Ck4b85yxdTyelsMUgFUXiw==", + "dev": true, + "license": "MIT", + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@yao-pkg/pkg/node_modules/@babel/types": { + "version": "7.23.0", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.23.0.tgz", + "integrity": "sha512-0oIyUfKoI3mSqMvsxBdclDwxXKXAUA8v/apZbc+iSyARYou1o8ZGDxbUYyLFoW2arqS2jDGqJuZvv1d/io1axg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^7.22.5", + "@babel/helper-validator-identifier": "^7.22.20", + "to-fast-properties": "^2.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@yao-pkg/pkg/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/@yao-pkg/pkg/node_modules/minimatch": { + "version": "9.0.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.4.tgz", + "integrity": "sha512-KqWh+VchfxcMNRAJjj2tnsSJdNbHsVgnkBhTNrW7AjVo6OvLtxw8zfT9oLw1JSohlFzJ8jCoTgaoXvJ+kHt6fw==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/abab": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.6.tgz", @@ -5376,39 +5506,6 @@ "node": ">= 6" } }, - "node_modules/pkg": { - "version": "5.8.1", - "resolved": "https://registry.npmjs.org/pkg/-/pkg-5.8.1.tgz", - "integrity": "sha512-CjBWtFStCfIiT4Bde9QpJy0KeH19jCfwZRJqHFDFXfhUklCx8JoFmMj3wgnEYIwGmZVNkhsStPHEOnrtrQhEXA==", - "dev": true, - "dependencies": { - "@babel/generator": "7.18.2", - "@babel/parser": "7.18.4", - "@babel/types": "7.19.0", - "chalk": "^4.1.2", - "fs-extra": "^9.1.0", - "globby": "^11.1.0", - "into-stream": "^6.0.0", - "is-core-module": "2.9.0", - "minimist": "^1.2.6", - "multistream": "^4.1.0", - "pkg-fetch": "3.4.2", - "prebuild-install": "7.1.1", - "resolve": "^1.22.0", - "stream-meter": "^1.0.4" - }, - "bin": { - "pkg": "lib-es5/bin.js" - }, - "peerDependencies": { - "node-notifier": ">=9.0.1" - }, - "peerDependenciesMeta": { - "node-notifier": { - "optional": true - } - } - }, "node_modules/pkg-dir": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", @@ -5421,98 +5518,6 @@ "node": ">=8" } }, - "node_modules/pkg-fetch": { - "version": "3.4.2", - "resolved": "https://registry.npmjs.org/pkg-fetch/-/pkg-fetch-3.4.2.tgz", - "integrity": "sha512-0+uijmzYcnhC0hStDjm/cl2VYdrmVVBpe7Q8k9YBojxmR5tG8mvR9/nooQq3QSXiQqORDVOTY3XqMEqJVIzkHA==", - "dev": true, - "dependencies": { - "chalk": "^4.1.2", - "fs-extra": "^9.1.0", - "https-proxy-agent": "^5.0.0", - "node-fetch": "^2.6.6", - "progress": "^2.0.3", - "semver": "^7.3.5", - "tar-fs": "^2.1.1", - "yargs": "^16.2.0" - }, - "bin": { - "pkg-fetch": "lib-es5/bin.js" - } - }, - "node_modules/pkg-fetch/node_modules/lru-cache": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "dev": true, - "dependencies": { - "yallist": "^4.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/pkg-fetch/node_modules/semver": { - "version": "7.6.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.0.tgz", - "integrity": "sha512-EnwXhrlwXMk9gKu5/flx5sv/an57AkRplG3hTK68W7FRDN+k+OWBj65M7719OkA82XLBxrcX0KSHj+X5COhOVg==", - "dev": true, - "dependencies": { - "lru-cache": "^6.0.0" - }, - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/pkg-fetch/node_modules/yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true - }, - "node_modules/pkg/node_modules/@babel/generator": { - "version": "7.18.2", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.18.2.tgz", - "integrity": "sha512-W1lG5vUwFvfMd8HVXqdfbuG7RuaSrTCCD8cl8fP8wOivdbtbIg2Db3IWUcgvfxKbbn6ZBGYRW/Zk1MIwK49mgw==", - "dev": true, - "dependencies": { - "@babel/types": "^7.18.2", - "@jridgewell/gen-mapping": "^0.3.0", - "jsesc": "^2.5.1" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/pkg/node_modules/@babel/parser": { - "version": "7.18.4", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.18.4.tgz", - "integrity": "sha512-FDge0dFazETFcxGw/EXzOkN8uJp0PC7Qbm+Pe9T+av2zlBpOgunFHkQPPn+eRuClU73JF+98D531UgayY89tow==", - "dev": true, - "bin": { - "parser": "bin/babel-parser.js" - }, - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/pkg/node_modules/@babel/types": { - "version": "7.19.0", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.19.0.tgz", - "integrity": "sha512-YuGopBq3ke25BVSiS6fgF49Ul9gH1x70Bcr6bqRLjWCkcX8Hre1/5+z+IiWOIerRMSSEfGZVB9z9kyq7wVs9YA==", - "dev": true, - "dependencies": { - "@babel/helper-string-parser": "^7.18.10", - "@babel/helper-validator-identifier": "^7.18.6", - "to-fast-properties": "^2.0.0" - }, - "engines": { - "node": ">=6.9.0" - } - }, "node_modules/prebuild-install": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.1.tgz", diff --git a/package.json b/package.json index 3858d05..f51ab99 100644 --- a/package.json +++ b/package.json @@ -12,10 +12,10 @@ "test-ci": "jest --ci", "start": "node --no-warnings src/server.js", "prepare-binary": "rm -rf dist && mkdir dist", - "create-win-x64-dist": "pkg package.json -t node16-win-x64 --out-path dist", - "create-mac-x64-dist": "pkg package.json -t node16-macos-x64 --out-path dist", - "create-linux-x64-dist": "pkg package.json -t node16-linux-x64 --out-path dist", - "create-linux-arm64-dist": "pkg package.json -t node16-linux-arm64 --out-path dist" + "create-win-x64-dist": "pkg package.json -t node20-win-x64 --out-path dist", + "create-mac-x64-dist": "pkg package.json -t node20-macos-x64 --out-path dist", + "create-linux-x64-dist": "pkg package.json -t node20-linux-x64 --out-path dist", + "create-linux-arm64-dist": "pkg package.json -t node20-linux-arm64 --out-path dist" }, "pkg": { "scripts": "package.json", @@ -54,11 +54,11 @@ "winston-daily-rotate-file": "^4.7.1" }, "devDependencies": { + "@yao-pkg/pkg": "^5.12.0", "chai": "^4.4.1", "cors": "^2.8.5", "jest": "^27.5.1", "nock": "^13.5.3", - "pkg": "^5.8.1", "proxyquire": "^2.1.3", "sinon": "^11.1.2", "sinon-chai": "^3.7.0" From 260a5522c792fa7cbe73f26ec26b82ae8ae18035 Mon Sep 17 00:00:00 2001 From: Zachary Brown Date: Fri, 9 Aug 2024 12:57:21 -0700 Subject: [PATCH 5/9] ci: update CI to node 20 --- .github/workflows/build.yaml | 4 ++-- .github/workflows/tests.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 1cc04a6..4b3c010 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -45,10 +45,10 @@ jobs: - name: Checkout Code uses: actions/checkout@v3 - - name: Setup Node 18.16 + - name: Setup Node 20.16 uses: actions/setup-node@v3 with: - node-version: '18.16' + node-version: '20.16' - name: Ignore Husky run: npm pkg delete scripts.prepare diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 05af295..b45921a 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -14,7 +14,7 @@ jobs: name: NPM Tests runs-on: ubuntu-latest container: - image: node:18.16 + image: node:20.16 steps: - uses: Chia-Network/actions/clean-workspace@main From 935f4f85a77aaa5f2c8ef42cca57a3cb3fa526f7 Mon Sep 17 00:00:00 2001 From: William Wills Date: Fri, 23 Aug 2024 18:27:37 -0400 Subject: [PATCH 6/9] feat: added organizations proxy --- src/proxy.js | 18 ++++++++++++++++++ src/server.js | 1 + 2 files changed, 19 insertions(+) diff --git a/src/proxy.js b/src/proxy.js index d4a00a7..826789a 100644 --- a/src/proxy.js +++ b/src/proxy.js @@ -90,6 +90,23 @@ const getUntokenizedUnits = () => { proxyReq.setHeader("x-api-key", CONFIG().CADT.API_KEY); } }, + }); +}; + +const getOrganizationsFromRegistry = () => { + return createProxyMiddleware({ + target: registryUri, + changeOrigin: true, + secure: false, + pathRewrite: async (path) => { + const homeOrgUid = await getHomeOrgUid(); + return "/v1/organizations"; + }, + onProxyReq: (proxyReq) => { + if (CONFIG().CADT.API_KEY) { + proxyReq.setHeader("x-api-key", CONFIG().CADT.API_KEY); + } + }, onProxyRes: async (proxyRes) => { const homeOrgUid = await getHomeOrgUid(); if (homeOrgUid) { @@ -104,4 +121,5 @@ module.exports = { getTokenizedUnits, getProjectsFromRegistry, getUntokenizedUnits, + getOrganizationsFromRegistry }; diff --git a/src/server.js b/src/server.js index da9cf22..7e598ec 100644 --- a/src/server.js +++ b/src/server.js @@ -47,6 +47,7 @@ app.use(errorHandler); app.use("/units/tokenized", proxy.getTokenizedUnits()); app.use("/projects", proxy.getProjectsFromRegistry()); app.use("/units/untokenized", proxy.getUntokenizedUnits()); +app.use("/organizations", proxy.getOrganizationsFromRegistry()); // Routes app.post("/tokenize", validator.body(tokenizeUnitSchema), tokenizeUnit); From 1a953fa6d79f3485b7f247147c2e388fcdb0bd56 Mon Sep 17 00:00:00 2001 From: William Wills Date: Wed, 18 Sep 2024 11:53:40 -0400 Subject: [PATCH 7/9] feat: added core-registry header --- src/middleware.js | 14 ++++++++++++++ src/proxy.js | 7 +++++++ src/server.js | 5 +++-- 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/src/middleware.js b/src/middleware.js index e1c559f..6f23aff 100644 --- a/src/middleware.js +++ b/src/middleware.js @@ -45,6 +45,19 @@ const setOrgUidHeader = async (req, res, next) => { next(); }; +/** + * Middleware to set the core registry mode header (t/f) + * @param {Object} req - The request object + * @param {Object} res - The response object + * @param {Function} next - The next middleware function + * @returns {Promise} + */ +const setCoreRegistryModeHeader = async (req, res, next) => { + res.header("Access-Control-Expose-Headers", "x-core-registry-mode"); + res.header("x-core-registry-mode", Boolean(CONFIG()?.GENERAL?.CORE_REGISTRY_MODE)); + next(); +}; + const assertApiKey = (req, res, next) => { if ( CONFIG().TOKENIZATION_ENGINE.API_KEY && @@ -94,4 +107,5 @@ module.exports = { setOrgUidHeader, assertApiKey, assertHomeOrgExists, + setCoreRegistryModeHeader }; diff --git a/src/proxy.js b/src/proxy.js index 826789a..fa29a2f 100644 --- a/src/proxy.js +++ b/src/proxy.js @@ -90,6 +90,13 @@ const getUntokenizedUnits = () => { proxyReq.setHeader("x-api-key", CONFIG().CADT.API_KEY); } }, + onProxyRes: async (proxyRes) => { + const homeOrgUid = await getHomeOrgUid(); + if (homeOrgUid) { + proxyRes.headers["Access-Control-Expose-Headers"] = "x-org-uid"; + proxyRes.headers["x-org-uid"] = homeOrgUid; + } + }, }); }; diff --git a/src/server.js b/src/server.js index 7e598ec..2620061 100644 --- a/src/server.js +++ b/src/server.js @@ -16,7 +16,7 @@ const { errorHandler, setOrgUidHeader, assertApiKey, - assertHomeOrgExists, + assertHomeOrgExists, setCoreRegistryModeHeader, } = require("./middleware"); const scheduler = require("./tasks"); @@ -40,6 +40,7 @@ app.use(cors()); app.use(assertHomeOrgExists); app.use(setOrgUidHeader); +app.use(setCoreRegistryModeHeader); app.use(assertApiKey); app.use(errorHandler); @@ -52,7 +53,7 @@ app.use("/organizations", proxy.getOrganizationsFromRegistry()); // Routes app.post("/tokenize", validator.body(tokenizeUnitSchema), tokenizeUnit); app.post("/parse-detok-file", parseDetokFile); -app.post("/confirm-detokanization", confirmDetokanization); +app.post("/confirm-detokenization", confirmDetokanization); /** * Basic health check route. From e618731e2f32eada5312d1da479c95d9f51b7ac6 Mon Sep 17 00:00:00 2001 From: Starttoaster Date: Tue, 24 Sep 2024 10:53:23 -0700 Subject: [PATCH 8/9] ci: Use GitHub Glue Action instead of curl --- .github/workflows/build.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 4b3c010..7479740 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -277,10 +277,10 @@ jobs: run: | echo "REPO_NAME=$(echo "$GITHUB_REPOSITORY" | cut -d "/" -f 2)" >>$GITHUB_OUTPUT - - name: Gets JWT Token from GitHub - uses: Chia-Network/actions/github/jwt@main - - name: Trigger apt repo update - run: | - curl -s -XPOST -H "Authorization: Bearer ${{ env.JWT_TOKEN }}" --data '{"climate_tokenization_repo":"${{ steps.repo-name.outputs.REPO_NAME }}","application_name":"[\"climate-tokenization-engine\"]","release_version":"${{ steps.tag-name.outputs.TAGNAME }}","add_debian_version":"true","arm64":"available"}' ${{ secrets.GLUE_API_URL }}/api/v1/climate-tokenization/${{ github.sha }}/start - curl -s -XPOST -H "Authorization: Bearer ${{ env.JWT_TOKEN }}" --data '{"climate_tokenization_repo":"${{ steps.repo-name.outputs.REPO_NAME }}","application_name":"[\"climate-tokenization-engine\"]","release_version":"${{ steps.tag-name.outputs.TAGNAME }}","add_debian_version":"true","arm64":"available"}' ${{ secrets.GLUE_API_URL }}/api/v1/climate-tokenization/${{ github.sha }}/success/deploy + uses: Chia-Network/actions/github/glue@main + with: + json_data: '{"climate_tokenization_repo":"${{ steps.repo-name.outputs.REPO_NAME }}","application_name":"[\"climate-tokenization-engine\"]","release_version":"${{ steps.tag-name.outputs.TAGNAME }}","add_debian_version":"true","arm64":"available"}' + glue_url: ${{ secrets.GLUE_API_URL }} + glue_project: "climate-tokenization" + glue_path: "trigger" From 1882d4108233c3c35af678bea5fa767b55e19571 Mon Sep 17 00:00:00 2001 From: Zachary Brown Date: Wed, 23 Oct 2024 11:26:28 -0700 Subject: [PATCH 9/9] chore: version bump --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index f51ab99..f0d215c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "climate-tokenization-engine", - "version": "1.3.21", + "version": "1.3.22", "bin": "./src/server.js", "description": "", "main": "proxy.js",