From 92c99c7e7969587292a011d6ca244c36d77023ed Mon Sep 17 00:00:00 2001 From: Romy <35330373+romayalon@users.noreply.github.com> Date: Tue, 15 Oct 2024 16:12:39 +0300 Subject: [PATCH] NC | Online upgrade process small fixes Signed-off-by: Romy <35330373+romayalon@users.noreply.github.com> --- src/cmd/nsfs.js | 57 +++++----------------- src/sdk/config_fs.js | 78 ++++++++++++++++++++++++++++++- src/upgrade/nc_upgrade_manager.js | 6 +++ src/upgrade/upgrade_utils.js | 1 + 4 files changed, 95 insertions(+), 47 deletions(-) diff --git a/src/cmd/nsfs.js b/src/cmd/nsfs.js index dc6a3171ea..3a2a5645d7 100644 --- a/src/cmd/nsfs.js +++ b/src/cmd/nsfs.js @@ -41,7 +41,6 @@ const BucketSpaceFS = require('../sdk/bucketspace_fs'); const SensitiveString = require('../util/sensitive_string'); const endpoint_stats_collector = require('../sdk/endpoint_stats_collector'); //const { RPC_BUFFERS } = require('../rpc'); -const pkg = require('../../package.json'); const AccountSDK = require('../sdk/account_sdk'); const AccountSpaceFS = require('../sdk/accountspace_fs'); const NoobaaEvent = require('../manage_nsfs/manage_nsfs_events_utils').NoobaaEvent; @@ -240,45 +239,6 @@ class NsfsAccountSDK extends AccountSDK { } } -async function init_nc_system(config_root) { - const config_fs = new ConfigFS(config_root); - const system_data = await config_fs.get_system_config_file({silent_if_missing: true}); - - const hostname = os.hostname(); - // If the system data already exists, we should not create it again - const updated_system_json = system_data || {}; - if (updated_system_json[hostname]?.current_version && updated_system_json.config_directory) return; - if (!updated_system_json[hostname]?.current_version) { - updated_system_json[hostname] = { - current_version: pkg.version, - upgrade_history: { successful_upgrades: [], last_failure: undefined } - }; - } - // If it's the first time a config_directory data is added to system.json - if (!updated_system_json.config_directory) { - updated_system_json.config_directory = { - config_dir_version: config_fs.config_dir_version, - upgrade_package_version: pkg.version, - phase: 'CONFIG_DIR_UNLOCKED', - upgrade_history: { successful_upgrades: [], last_failure: undefined } - }; - } - try { - if (system_data) { - await config_fs.update_system_config_file(JSON.stringify(updated_system_json)); - console.log('updated NC system data with version: ', pkg.version); - } else { - await config_fs.create_system_config_file(JSON.stringify(updated_system_json)); - console.log('created NC system data with version: ', pkg.version); - } - } catch (err) { - const msg = 'failed to create/update NC system data due to - ' + err.message; - const error = new Error(msg); - console.error(msg, err); - throw error; - } -} - async function main(argv = minimist(process.argv.slice(2))) { try { config.DB_TYPE = 'none'; @@ -313,11 +273,6 @@ async function main(argv = minimist(process.argv.slice(2))) { const versioning = argv.versioning || 'DISABLED'; const fs_root = argv._[0] || ''; - // Do not move this function - we need to update RPM changes before starting the endpoint - const config_fs = new ConfigFS(nsfs_config_root); - const nc_upgrade_manager = new NCUpgradeManager(config_fs); - await nc_upgrade_manager.update_rpm_upgrade(); - const fs_config = { uid, gid, @@ -363,7 +318,17 @@ async function main(argv = minimist(process.argv.slice(2))) { nsfs_config_root, }); - if (!simple_mode) await init_nc_system(nsfs_config_root); + if (!simple_mode) { + // Do not move this function - we need to create/update RPM changes before starting the endpoint + const config_fs = new ConfigFS(nsfs_config_root); + const system_data = await config_fs.get_system_config_file({ silent_if_missing: true }); + if (system_data && system_data[os.hostname()]) { + const nc_upgrade_manager = new NCUpgradeManager(config_fs); + await nc_upgrade_manager.update_rpm_upgrade(); + } else { + await config_fs.init_nc_system(); + } + } const endpoint = require('../endpoint/endpoint'); await endpoint.main({ diff --git a/src/sdk/config_fs.js b/src/sdk/config_fs.js index e4cb6ea81f..29f5ce2fcf 100644 --- a/src/sdk/config_fs.js +++ b/src/sdk/config_fs.js @@ -1,11 +1,13 @@ /* Copyright (C) 2024 NooBaa */ 'use strict'; +const os = require('os'); const util = require('util'); const _ = require('lodash'); const path = require('path'); const P = require('../util/promise'); const config = require('../../config'); +const pkg = require('../../package.json'); const dbg = require('../util/debug_module')(__filename); const os_utils = require('../util/os_utils'); const SensitiveString = require('../util/sensitive_string'); @@ -1026,6 +1028,41 @@ class ConfigFS { } } + /** + * init_nc_system creates/updates system.json file + * if system.json does not exist (a new system) - host and config dir data will be set on the newly created file + * else - + * 1. if the host data already exist in system.json - return + * 2. set the host data on system.json data and update the file + * Note - config directory data on upgraded systems will be set by nc_upgrade_manager + * @returns + */ + async init_nc_system() { + const system_data = await this.get_system_config_file({silent_if_missing: true}); + + let updated_system_json = system_data || {}; + const is_new_system = !system_data; + const hostname = os.hostname(); + try { + if (is_new_system) { + updated_system_json = this._get_new_system_json_data(); + await this.create_system_config_file(JSON.stringify(updated_system_json)); + dbg.log0('created NC system data with version: ', pkg.version); + } else { + if (updated_system_json[hostname]?.current_version) return; + const new_host_data = this._get_new_hostname_data(); + updated_system_json = { ...updated_system_json, new_host_data }; + await this.update_system_config_file(JSON.stringify(updated_system_json)); + dbg.log0('updated NC system data with version: ', pkg.version); + } + } catch (err) { + const msg = 'failed to create/update NC system data due to - ' + err.message; + const error = new Error(msg); + dbg.error(msg, err); + throw error; + } + } + //////////////////////// /// HELPERS //// //////////////////////// @@ -1093,8 +1130,13 @@ class ConfigFS { * @returns {Promise} */ async _throw_if_config_dir_locked() { - const system_data = await this.get_system_config_file({silent_if_missing: true}); + const system_data = await this.get_system_config_file({ silent_if_missing: true }); + // if system was never created, currently we allow using the CLI without creating system + // we should consider changing it to throw on this scenario as well if (!system_data) return; + if (!system_data.config_directory) { + throw new RpcError('CONFIG_DIR_VERSION_MISMATCH', `config_directory data is missing in system.json, any updates to the config directory are blocked until the config dir upgrade`); + } const running_code_config_dir_version = this.config_dir_version; const system_config_dir_version = system_data.config_directory.config_dir_version; const ver_comparison = version_compare(running_code_config_dir_version, system_config_dir_version); @@ -1107,6 +1149,40 @@ class ConfigFS { `mentioned in system.json =${system_config_dir_version}, any updates to the config directory are blocked until the source code upgrade`); } } + + /** + * _get_new_hostname_data returns new hostanme data for system.json + * @returns {Object} + */ + _get_new_hostname_data() { + return { + [os.hostname()]: { + current_version: pkg.version, + upgrade_history: { + successful_upgrades: [] + }, + }, + }; + } + + /** + * _get_new_system_json_data returns new system.json data + * @returns {Object} + */ + _get_new_system_json_data() { + return { + ...this._get_new_hostname_data(), + config_directory: { + config_dir_version: this.config_dir_version, + upgrade_package_version: pkg.version, + phase: 'CONFIG_DIR_UNLOCKED', + upgrade_history: { + successful_upgrades: [], + last_failure: undefined + } + } + }; + } } // EXPORTS diff --git a/src/upgrade/nc_upgrade_manager.js b/src/upgrade/nc_upgrade_manager.js index dab662e92c..83bde5fa57 100644 --- a/src/upgrade/nc_upgrade_manager.js +++ b/src/upgrade/nc_upgrade_manager.js @@ -137,6 +137,7 @@ class NCUpgradeManager { /** * config_directory_defaults returns a default initial config directory object * @param {Object} system_data + * @returns {Object} */ config_directory_defaults(system_data) { const hosts_old_package_version = system_data?.[hostname]?.upgrade_history?.successful_upgrades?.[0]?.from_version; @@ -241,6 +242,7 @@ class NCUpgradeManager { /** * _run_nc_upgrade_scripts runs the config directory upgrade scripts * @param {Object} this_upgrade + * @returns {Promise} */ async _run_nc_upgrade_scripts(this_upgrade) { try { @@ -259,6 +261,7 @@ class NCUpgradeManager { * 3. upgrade_package_version is the new source code version * 4. add the finished upgrade to the successful_upgrades array * @param {Object} system_data + * @param {Object} this_upgrade * @returns {Promise} */ async _update_config_dir_upgrade_finish(system_data, this_upgrade) { @@ -281,6 +284,9 @@ class NCUpgradeManager { /** * _update_config_dir_upgrade_failed updates the system.json on failure of the upgrade * @param {Object} system_data + * @param {Object} this_upgrade + * @param {Error} error + * @returns {Promise} */ async _update_config_dir_upgrade_failed(system_data, this_upgrade, error) { system_data.config_directory.upgrade_history.last_failure = this_upgrade; diff --git a/src/upgrade/upgrade_utils.js b/src/upgrade/upgrade_utils.js index b8fa6ad653..453548eb1c 100644 --- a/src/upgrade/upgrade_utils.js +++ b/src/upgrade/upgrade_utils.js @@ -67,6 +67,7 @@ function should_upgrade(server_version, container_version) { * load_required_scripts loads all scripts that should be run according to the given versions * @param {string} server_version * @param {string} container_version + * @param {string} upgrade_scripts_dir */ async function load_required_scripts(server_version, container_version, upgrade_scripts_dir) { // expecting scripts directories to be in a semver format. e.g. ./upgrade_scripts/5.0.1