diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..48a88e7 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 ARROW & BOW + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..158ecfa --- /dev/null +++ b/README.md @@ -0,0 +1,19 @@ +# BOW: Bayesian-Optimized Wavelengths + +## 1. Overview +BOW is a bayesian optimization system that optimizes optical wavelengths' Quality of Transmission (QoT) metrics (e.g., OSNR) for wavelength reconfigurations. BOW is built on Python 3.8.8, with Ax (https://ax.dev) as the Bayesian Optimization backend, GNPy (https://gnpy.readthedocs.io/en/master/) as the optical-layer QoT estimator, and FCR (https://github.com/facebookincubator/FCR) as the control interface to optical network devices. + +For a full technical description on BOW, please read our OFC 2021 paper: + +> Z. Zhong, M. Ghobadi, M. Balandat, S. Katti, A. Kazerouni, J. Leach, M. McKillop, Y. Zhang, "BOW: First Real-World Demonstration of a Bayesian Optimization System for Wavelength Reconfiguration," OFC, 2021. http://bow.csail.mit.edu/files/OFC-21-BOW-final.pdf + +For more details on BOW, please visit our website: http://bow.csail.mit.edu + + +## 2. Requirement +* Python 3.8 +* Ax 0.1.20 +* GNPy 2.1 + +## 3. Licence +BOW is MIT-licenced. diff --git a/bayesian_amp_control.py b/bayesian_amp_control.py new file mode 100644 index 0000000..0f2ee8e --- /dev/null +++ b/bayesian_amp_control.py @@ -0,0 +1,867 @@ +import concurrent.futures +import json +import math +import random +import time +from collections import defaultdict +import matplotlib.pyplot as plt +import pyjq +from ax.modelbridge.generation_strategy import GenerationStep, GenerationStrategy +from ax.modelbridge.registry import Models +from ax.service.ax_client import AxClient +from crypto.keychain_service.keychain import ttypes as keychain +from fbnet.command_runner.CommandRunner.Command import Client as FcrClient +from fbnet.command_runner.CommandRunner.ttypes import Device +from fbnet.command_runner.CommandRunner.ttypes import SessionType, SessionData +from libfb.py.thrift import get_sr_client +from libfb.py.thrift_clients.keychain_thrift_client import KeychainClient +from gnpy.core.utils import db2lin, lin2db + +from fcr_interface import clean_transponder +from input_parameters import amp_type_id +from plotting import plot_spectrum +from simulator import run_simulator +from ssh_connection import ( + ssh_setup, + ssh_close, +) + + +FCR_PROD_TIER = "netsystems.fbnet_command_runner" + + +def get_secret(name, group): + req = keychain.GetSecretRequest( + name=name, + group=group, + ) + try: + return KeychainClient().getSecret(req) + except keychain.KeychainServiceException as ex: + print("Error retrieving secret:" + ex) + return False + + +def run_command(device_commands, read_or_write): + try: + user = get_secret("TACACS_USERNAME", "NETENG_AUTOMATION").secret + pw = get_secret("TACACS_PASSWORD", "NETENG_AUTOMATION").secret + if read_or_write == 1: + persistent_sign = "auto" + else: + persistent_sign = "auto" + fcr_commands = { + Device( + hostname=device, + username=user, + password=pw, + session_type=SessionType.TL1, + session_data=SessionData( + extra_options={ + "format": "json", + "use_persistent_connection": persistent_sign, + } + ), + ): device_commands[device] + for device in device_commands + } + with get_sr_client( + FcrClient, + tier=FCR_PROD_TIER, + ) as client: + res = client.bulk_run(fcr_commands) + timestamp = time.localtime() + timestamp = time.strftime("%Y.%m.%d.%H.%M.%S", timestamp) + return res, timestamp + + except Exception as ex: + print("User exception: {}".format(ex)) + timestamp = time.localtime() + timestamp = time.strftime("%Y.%m.%d.%H.%M.%S", timestamp) + return 1, timestamp + + +def read_proprietary_controller(proprietary_controller_node, proprietary_controller_id): + read_proprietary_controllerler_cmd = "XXXXXXXXXXX" ## anonymized southbound command + + commands = defaultdict(list) + commands[proprietary_controller_node] = [read_proprietary_controller_cmd] + + results = 1 + count = 0 + while results == 1 and count < 10: # rc = 1 means exception + results, timestamp = run_command(commands, read_or_write=1) + count += 1 + + if results == 1: + print("the FCR fails to fetch data") + stat = "N/A" + return stat, timestamp + else: + print(results[proprietary_controller_node]) + vars = {} + if results[proprietary_controller_node][0].output == "": + stat = "N/A" + else: + j_data = json.loads(results[proprietary_controller_node][0].output) + jqscript1 = ".[][].fields[3]" + primary = pyjq.all(jqscript1, j_data, vars=vars) + print("proprietary_controller primary state:", primary[0]) + stat = primary[0] + + return stat, timestamp + + +def control_proprietary_controller(proprietary_controller_node, proprietary_controller_id, on_off_flag): + enable_proprietary_controller_cmd = "XXXXXXXXXXX" ## anonymized southbound command + dis_proprietary_controller_cmd = "XXXXXXXXXXX" ## anonymized southbound command + + commands = defaultdict(list) + + if on_off_flag == 1: + commands[proprietary_controller_node] = [enable_proprietary_controller_cmd] + print(commands) + res, timestamp = run_command(commands, read_or_write=0) + + else: + commands[proprietary_controller_node] = [dis_proprietary_controller_cmd] + print(commands) + res, timestamp = run_command(commands, read_or_write=0) + + return res, timestamp + + +def optimize_proprietary_controller(proprietary_controller_node, proprietary_controller_id): + reopt_proprietary_controller_cmd = "XXXXXXXXXXX" ## anonymized southbound command + + commands = defaultdict(list) + commands[proprietary_controller_node] = [reopt_proprietary_controller_cmd] + print(commands) + res, timestamp = run_command(commands, read_or_write=0) + + return res, timestamp + + +def WSS_ONOFF_bulkctl(node_id, ONOFF_names, opqstatus): + ctl_WSS_ONOFF = "XXXXXXXXXXX" ## anonymized southbound command + current_cmds = [] + for current_name in ONOFF_names: + cmd = ctl_WSS_ONOFF.replace("ONOFF_ID", current_name) + cmd = cmd.replace("OPQ_STATUS", opqstatus[ONOFF_names.index(current_name)]) + current_cmds.append(cmd) + + commands = defaultdict(list) + commands[node_id] = current_cmds + + rc, _ = run_command(commands, read_or_write=0) + + return rc + + +def proprietary_controller_prepareness(proprietary_controller_node, proprietary_controller_id, final_state, verbose): + readings, timestamp = read_proprietary_controller(proprietary_controller_node, proprietary_controller_id) + if verbose: + print("read_proprietary_controller command 1", readings, timestamp) + # the final target is to disable proprietary_controller for BO amp configuration + if final_state == "OUT_OF_SERVICE": + if readings == "OUT_OF_SERVICE": + res, timestamp = control_proprietary_controller(proprietary_controller_node, proprietary_controller_id, 1) + if verbose: + print("control_proprietary_controller command 2", res, timestamp) + time.sleep(20) + res, timestamp = optimize_proprietary_controller(proprietary_controller_node, proprietary_controller_id) + if verbose: + print("optimize_proprietary_controller command 3", res, timestamp) + time.sleep(20) + res, timestamp = control_proprietary_controller(proprietary_controller_node, proprietary_controller_id, 0) + if verbose: + print("control_proprietary_controller command 4", res, timestamp) + time.sleep(20) + else: + res, timestamp = optimize_proprietary_controller(proprietary_controller_node, proprietary_controller_id) + if verbose: + print("optimize_proprietary_controller command 2", res, timestamp) + time.sleep(20) + res, timestamp = control_proprietary_controller(proprietary_controller_node, proprietary_controller_id, 0) + if verbose: + print("control_proprietary_controller command 3", res, timestamp) + time.sleep(20) + # the final target is to enable proprietary_controller for channel on off state config + else: + if readings == "OUT_OF_SERVICE": + res, timestamp = control_proprietary_controller(proprietary_controller_node, proprietary_controller_id, 1) + if verbose: + print("control_proprietary_controller command 2", res, timestamp) + time.sleep(20) + res, timestamp = optimize_proprietary_controller(proprietary_controller_node, proprietary_controller_id) + if verbose: + print("optimize_proprietary_controller command 3", res, timestamp) + time.sleep(20) + else: + res, timestamp = optimize_proprietary_controller(proprietary_controller_node, proprietary_controller_id) + if verbose: + print("optimize_proprietary_controller command 2", res, timestamp) + time.sleep(20) + return + + +def get_line_osnr( + line, ssh_list, estimated_channel_osnr, estimated_channel_power, plot_flag, ssh_flag +): + # read the performance metrics on related devices after amplifier change + ( + new_spectrum, + new_noise, + central_freq, + fine_grain_spectrum, + fine_grain_frequency, + bins, + ) = line.read_spectrum(line.wss, ssh_list, ssh_flag) + + if plot_flag: + plot_spectrum( + line, + new_spectrum, + new_noise, + central_freq, + fine_grain_spectrum, + fine_grain_frequency, + bins, + estimated_channel_power, + ) + + current_osnr = {} + for tx in line.transponder_fre: + current_osnr[line.transponder_fre[tx]] = None # initiate osnr dict + + for ch in range(len(new_spectrum)): + if new_spectrum[ch] > -15: # regular channel power should be larger than -15 + if ( + new_spectrum[ch] - new_noise[ch] > 8 + ): # channel should have larger than 8 db osnr + current_osnr[central_freq[ch]] = round( + (new_spectrum[ch] - new_noise[ch]), 2 + ) # IEC OSNR for coarsely-distributed channels with measurable noise floors + # current_osnr[central_freq[ch]] = round( + # 10*math.log10((db2lin(new_spectrum[ch]) - db2lin(new_noise[ch])) / (db2lin(new_noise[ch])/2)), + # 2, + # ) # pol-mux OSNR estimation for densely-distributed channels with non-measurable noise floors + else: + print("Error! very weak channel signal") + + else: + print("Error! very weak channel signal") + current_osnr[central_freq[ch]] = round( + (new_spectrum[ch] - new_noise[ch]), 2 + ) + + return current_osnr + + +def PrintTime(print_flag): + timestamp = time.localtime() + pt_time = time.strftime("%Y.%m.%d.%H.%M.%S", timestamp) + print(print_flag, pt_time) + return timestamp + + +def Evaluate( + line, + parameters, + ssh_list, + estimated_channel_osnr, + estimated_channel_power, + delay_vector, + write_sign, + metric_sign, + ssh_flag, +): ## evaluate the generated parameter and return reward for next acquisition + ts0 = PrintTime("Evaluation begins Timestamp") + # clear transponder measurement bin + if line.fast_slow_version == "slow": + clean_transponder(line.transponder_box) + # write the amplifier parameters to the devices + if line.fast_slow_version == "slow": + time.sleep(5) + ts1 = time.localtime() + ts2 = time.localtime() + if write_sign: + ts1 = PrintTime("write_amp begins Timestamp") + line.write_amp(parameters, ssh_list, ssh_flag) + ts2 = PrintTime("write_amp ends Timestamp") + if line.fast_slow_version == "slow": + time.sleep(15) + if line.fast_slow_version == "slow": + metric_bit = [1, 1, 1, 1, 1, 1] + else: + metric_bit = [1, 0, 1, 0, 1, 0] + + transponder_Q_set = {} + transponder_ber_set = {} + transponder_esnr_set = {} + + # read line OSNR and transponder metric in parallel + with concurrent.futures.ThreadPoolExecutor() as executor: + ts3 = PrintTime("get_line_osnr begins Timestamp") + osnr = executor.submit( + get_line_osnr, + line, + ssh_list, + estimated_channel_osnr, + estimated_channel_power, + plot_flag=1, + ssh_flag=ssh_flag, + ) + current_osnr = osnr.result() + ts5 = PrintTime("get_line_osnr ends Timestamp") + + # we want to maximize the min OSNR first, then the OSNR of all channels to be as large as possible + reward = 0 + if metric_sign == 0: # maximize for OSNR + if len(current_osnr) > 0: + reward = min(current_osnr.values()) * 10000 + for c in current_osnr: + reward += current_osnr[c] + + elif metric_sign == 1: # maximize for Q + if len(transponder_Q_set) > 0: + reward = min(transponder_Q_set.values()) * 10000 + for c in transponder_Q_set: + reward += transponder_Q_set[c] + + else: # minimize for BER + if len(transponder_ber_set) > 0: + reward = max(transponder_ber_set.values()) * 10000 + for c in transponder_ber_set: + reward += transponder_ber_set[c] + + return_reward = {"bow": (reward, None)} + ts7 = PrintTime("Evaluation finish Timestamp") + + write_delay = time.mktime(ts2) - time.mktime(ts1) + read_osnr_delay = time.mktime(ts5) - time.mktime(ts3) + read_transponder_delay = 0 # no need to read transponder here + total_delay = time.mktime(ts7) - time.mktime(ts0) + delay_vector.append(write_delay) + delay_vector.append(read_osnr_delay) + delay_vector.append(read_transponder_delay) + delay_vector.append(total_delay) + + return ( + return_reward, + current_osnr, + transponder_ber_set, + transponder_Q_set, + transponder_esnr_set, + delay_vector, + ) + + +def SafeCheck(parameters, amp_type_id, fiber_spans, transponder_fre, verbose): + print("\033[1;32m Run Safecheck\033[0;0m") + safecheck_reward = 1 + left_freq = 999999999 + right_freq = 0 + for tx in transponder_fre: + if transponder_fre[tx] < left_freq: + left_freq = transponder_fre[tx] + if transponder_fre[tx] > right_freq: + right_freq = transponder_fre[tx] + ts1 = PrintTime("simulator starts Timestamp") + + ## run GNPy simulator 10 times and take average + estimated_channel_osnr = {} + estimated_channel_power = {} + for _ in range(10): + estimated_channel_osnr_current, estimated_channel_power_current = run_simulator( + parameters, + amp_type_id, + fiber_spans, + left_freq - 37500, + right_freq + 37500, + verbose=verbose, + ) # 4800 GHz C-band spectrum, @75 GHz spacing + for ch in estimated_channel_osnr_current: + if len(estimated_channel_osnr): + estimated_channel_osnr[ch] += estimated_channel_osnr_current[ch] + estimated_channel_power[ch] += estimated_channel_power_current[ch] + else: + estimated_channel_osnr[ch] = estimated_channel_osnr_current[ch] + estimated_channel_power[ch] = estimated_channel_power_current[ch] + for ch in estimated_channel_osnr: + estimated_channel_osnr[ch] = estimated_channel_osnr[ch]/10 + print("GNPy estimated_channel_osnr", estimated_channel_osnr) + ts2 = PrintTime("simulator ends Timestamp") + simulator_delay = time.mktime(ts2) - time.mktime(ts1) + print("simulator_delay", simulator_delay) + + min_estimated_osnr = 999 + for ch in estimated_channel_osnr: + if min_estimated_osnr > estimated_channel_osnr[ch]: + min_estimated_osnr = estimated_channel_osnr[ch] + safecheck_reward = min_estimated_osnr * 10000 + + return safecheck_reward, estimated_channel_osnr, estimated_channel_power + + +def BayesianOptimization( + parameter_history, + reward_history, + osnr_history, + ber_history, + q_history, + esnr_history, + write_delay_history, + read_osnr_delay_history, + read_transponder_delay_history, + bo_acquisition_delay_history, + total_delay_history, + ax_parameters, + param_constraints, + fiber_spans, + amp_type_id, + line, + sobol_num, + gpei_num, + metric_sign, + ssh_flag, + ssh_list, + user, + verbose, +): + """ + The Bayesian Optimization iterative search loop + """ + ## Ax BO kernal settings + strategy = GenerationStrategy( + name="Sobol", + steps=[ + GenerationStep(model=Models.SOBOL, num_trials=sobol_num), + GenerationStep(model=Models.GPEI, num_trials=-1), + ], + ) + ax_client = AxClient(generation_strategy=strategy) + ax_client.create_experiment( + name="bow_test_experiment", + parameters=ax_parameters, + objective_name="bow", + minimize=False, + parameter_constraints=param_constraints, + experiment_type="network_systems", + ) + ## adaptive stopping criteria + current_best = 0 + increase_ratio = 0 + stopping_improvement = 0.05 + stopping_criteria = gpei_num + stopping = stopping_criteria # if best reward keeps increase less than 0.05 for 10 trials , stop + initial_reward = reward_history[0] + print(reward_history) + print( + "The initial reward value (threshold simulator/practical) is", + initial_reward["bow"][0], + ) + + # start BO iterations + try: + print( + "\033[1;34m ==========\n Start Bayesian Optimization\n ==========\033[0;0m" + ) + timestamp = time.localtime() + pt_time = time.strftime("%Y.%m.%d.%H.%M.%S", timestamp) + print("Timestamp is:", pt_time) + while stopping > 0: + ts1 = PrintTime("get_next_trial starts Timestamp") + parameters, trial_index = ax_client.get_next_trial() + ts2 = PrintTime("get_next_trial ends Timestamp") + get_next_trial_delay = time.mktime(ts2) - time.mktime(ts1) + print("\033[1;31m get_next_trial_delay", get_next_trial_delay, "\033[0;0m") + delay_vector = [get_next_trial_delay] + + print("\033[1;34m ==== BO loop", trial_index, "====\033[0;0m") + for x in parameters: + parameters[x] = round(parameters[x], 2) + if line.fast_slow_version == "slow": + print("sleep 10 before evaluating the parameters") + time.sleep(10) + # push parameter history + parameter_history.append(parameters) + # safety check by the gnpy simulator + ( + safecheck_reward_value, + estimated_channel_osnr, + estimated_channel_power, + ) = SafeCheck(parameters, amp_type_id, fiber_spans, line.transponder_fre, verbose) + + # the BO generated parameters are safe to deploy (min OSNR should be no smaller than initial state) + if ( + safecheck_reward_value > initial_reward["bow"][0] - 10000 + ): # assume 1dB margin + print( + "\033[1;32m SafeCheck passed, parameter safe, deploy parameter to network\033[0;0m" + ) + # evaluate the current parameters generated by BO + ( + current_reward, + current_osnr, + current_ber, + current_q, + current_esnr, + delay_vector, + ) = Evaluate( + line, + parameters, + ssh_list, + estimated_channel_osnr, + estimated_channel_power, + delay_vector, + write_sign=1, + metric_sign=metric_sign, + ssh_flag=ssh_flag, + ) + + print("current_osnr", current_osnr) + print("current Q", current_q) + print("current BER", current_ber) + print("current ESNR", current_esnr) + reward_history.append(current_reward) + osnr_history.append(current_osnr) + ber_history.append(current_ber) + q_history.append(current_q) + esnr_history.append(current_esnr) + + bo_acquisition_delay_history.append(delay_vector[0]) + write_delay_history.append(delay_vector[1]) + read_osnr_delay_history.append(delay_vector[2]) + read_transponder_delay_history.append(delay_vector[3]) + total_delay_history.append(delay_vector[4] + delay_vector[0]) + + ts3 = PrintTime("complete_trial starts Timestamp") + ax_client.complete_trial( + trial_index=trial_index, raw_data=current_reward + ) + ts4 = PrintTime("complete_trial ends Timestamp") + complete_trial_delay = time.mktime(ts4) - time.mktime(ts3) + print("complete_trial_delay", complete_trial_delay) + + # the BO generated parameters are NOT safe to deploy + else: + print( + "\033[1;32m SafeCheck failed, parameter not safe, so feedback simulator response only\033[0;0m" + ) + bo_acquisition_delay_history.append(delay_vector[0]) + write_delay_history.append(0) + read_osnr_delay_history.append(0) + read_transponder_delay_history.append(0) + total_delay_history.append(0) + + ( + current_reward, + current_osnr, + current_ber, + current_q, + current_esnr, + delay_vector, + ) = Evaluate( + line, + parameters, + ssh_list, + estimated_channel_osnr, + estimated_channel_power, + delay_vector, + write_sign=0, + metric_sign=metric_sign, + ssh_flag=ssh_flag, + ) + + print("current_osnr", current_osnr) + print("current Q", current_q) + print("current BER", current_ber) + print("current ESNR", current_esnr) + reward_history.append(current_reward) + osnr_history.append(current_osnr) + ber_history.append(current_ber) + q_history.append(current_q) + esnr_history.append(current_esnr) + + safecheck_reward = {"bow": (safecheck_reward_value, None)} + ax_client.complete_trial( + trial_index=trial_index, raw_data=safecheck_reward + ) + + # BO iteration termination condition: reward does not get improved for STOPPING_CRITERIA + if safecheck_reward_value > current_best: + if current_best > 0: + increase_ratio = float( + (safecheck_reward_value - current_best) / current_best + ) + current_best = safecheck_reward_value + if increase_ratio > stopping_improvement: + stopping = stopping_criteria + elif trial_index > sobol_num: + stopping = stopping - 1 + # if it is worse than history best, then gradually close loop + elif trial_index > sobol_num: + stopping = stopping - 1 + + except (Exception, KeyboardInterrupt) as ex: # catch *all* exceptions + print("Exceptions", ex) + for d in ssh_list: + line.ssh_cancel_user(user, ssh_list[d]) + ssh_close(ssh_list[d]) + print("Due to exception, Close ssh connection", ssh_list[d], "to device", d) + + # find the best parameters across all BO runs + best_parameters, values = ax_client.get_best_parameters() + for x in best_parameters: + best_parameters[x] = round(best_parameters[x], 2) + + print("best_parameters", best_parameters, values) + + JSON_LOC = "/home/zhizhenzhong/local/database/bo_json/" + timestamp = time.localtime() + pt_time = time.strftime("%Y.%m.%d.%H.%M.%S", timestamp) + JSONFILE = "ax_" + pt_time + ".json" + jsonfile_name = JSON_LOC + JSONFILE + ax_client.save_to_json_file(jsonfile_name) + + # config the best parameter, and read output + ( + safecheck_reward_value, + estimated_channel_osnr, + estimated_channel_power, + ) = SafeCheck(best_parameters, amp_type_id, fiber_spans, line.transponder_fre, verbose) + + delay_vector = [0] # this is for deploying the best parameter, no BO anymore + final_reward, final_osnr, final_ber, final_q, final_esnr, delay_vector = Evaluate( + line, + best_parameters, + ssh_list, + estimated_channel_osnr, + estimated_channel_power, + delay_vector, + write_sign=1, + metric_sign=0, + ssh_flag=ssh_flag, + ) + + reward_history.append(final_reward) + osnr_history.append(final_osnr) + ber_history.append(final_ber) + q_history.append(final_q) + esnr_history.append(final_esnr) + bo_acquisition_delay_history.append(delay_vector[0]) + write_delay_history.append(delay_vector[1]) + read_osnr_delay_history.append(delay_vector[2]) + read_transponder_delay_history.append(delay_vector[3]) + total_delay_history.append(delay_vector[4] + delay_vector[0]) + + ## close all ssh connections + for d in ssh_list: + print("cancel user", d) + line.ssh_cancel_user(user, ssh_list[d]) + ssh_close(ssh_list[d]) + print("Close ssh connection", ssh_list[d], "to device", d) + + return ( + best_parameters, + parameter_history, + reward_history, + osnr_history, + ber_history, + q_history, + esnr_history, + write_delay_history, + read_osnr_delay_history, + read_transponder_delay_history, + bo_acquisition_delay_history, + total_delay_history, + ) + + +def WavelengthReconfiguration( + ax_parameters, + param_constraints, + fiber_spans, + amp_type_id, + line, + sobol_num, + gpei_num, + metric_sign, + random_range, + ssh_flag, + verbose, +): + """ + The entire wavelength reconfiguration process including BO + """ + parameter_history = [] + reward_history = [] + osnr_history = [] + ber_history = [] + q_history = [] + esnr_history = [] + write_delay_history = [] + read_osnr_delay_history = [] + read_transponder_delay_history = [] + bo_acquisition_delay_history = [] + total_delay_history = [] + + # make sure the proprietary_controller is optimized and stay in service + proprietary_controller_prepareness( + line.proprietary_controller_info["location_proprietary_controller"], + line.proprietary_controller_info["proprietary_controller_id"], + final_state="INSERVICE", + verbose=verbose, + ) + time.sleep(20) + # create scenario where only partial channels present + WSS_ONOFF_bulkctl( + line.wss_onoff["node_id"], line.wss_onoff["ONOFF_name"], line.wss_onoff["a_opqstatus"] + ) + print("before provisioning:") + time.sleep(20) # wait for the WSS control command to take effect + get_line_osnr( + line, + ssh_list={}, + estimated_channel_osnr={}, + estimated_channel_power={}, + plot_flag=1, + ssh_flag=0, + ) + + # optimize proprietary_controller under partial channels, and turn off proprietary_controller + proprietary_controller_prepareness( + line.proprietary_controller_info["location_proprietary_controller"], + line.proprietary_controller_info["proprietary_controller_id"], + final_state="OUT_OF_SERVICE", + verbose=verbose, + ) + print("\033[1;31m Now we emulate a fiber cut event, with proprietary_controller disabled\033[0;0m") + time.sleep(10) + print("Clear transponder parameters first") + clean_transponder(line.transponder_box) + time.sleep(120) + + # turn channel on to emulate wavelength provisioning + WSS_ONOFF_bulkctl( + line.wss_onoff["node_id"], line.wss_onoff["ONOFF_name"], line.wss_onoff["z_opqstatus"] + ) + print("after provisioning:") + + # decide ssh method + ssh_list = {} + user = "0" + if ssh_flag == 1: + ## last is the osnr roadm, first is the controller roadm + ssh_devices = list(line.amplifiers.keys()) + ## setup dedicated ssh sockets for each device + for d in ssh_devices: + ssh = ssh_setup(d) + ssh_list[d] = ssh + time.sleep(5) # wait to make sure ssh connection is up + user = line.ssh_act_user(ssh) + print("ssh_list", ssh_list) + + initial_gain_results = line.read_amp(line.amplifiers) + + # Read initial parameter status and store in the correct form + initial_gain_results_float = [] + initial_gain_results_key = [] + for x in initial_gain_results: + initial_gain_results_float.append(float(initial_gain_results[x])) + initial_gain_results_key.append(x) + initial_gain_results_zip = dict( + zip(initial_gain_results_key, initial_gain_results_float) + ) + parameter_history.append(initial_gain_results_zip) + + # initial reward value + ( + current_reward, + current_osnr, + current_ber, + current_q, + current_esnr, + delay_vector, + ) = Evaluate( + line, + initial_gain_results, + ssh_list, + estimated_channel_osnr={}, + estimated_channel_power={}, + delay_vector=[0], + write_sign=0, + metric_sign=metric_sign, + ssh_flag=ssh_flag, # use FCR + ) + print("initial reward", current_reward) + print("initial osnr", current_osnr) + print("initial Q", current_q) + print("initial BER", current_ber) + print("initial ESNR", current_esnr) + reward_history.append(current_reward) + osnr_history.append(current_osnr) + ber_history.append(current_ber) + q_history.append(current_q) + esnr_history.append(current_esnr) + + bo_acquisition_delay_history.append(delay_vector[0]) + write_delay_history.append(delay_vector[1]) + read_osnr_delay_history.append(delay_vector[2]) + read_transponder_delay_history.append(delay_vector[3]) + total_delay_history.append(delay_vector[4] + delay_vector[0]) + + ( + best_parameters, + parameter_history, + reward_history, + osnr_history, + ber_history, + q_history, + esnr_history, + write_delay_history, + read_osnr_delay_history, + read_transponder_delay_history, + bo_acquisition_delay_history, + total_delay_history, + ) = BayesianOptimization( + parameter_history, + reward_history, + osnr_history, + ber_history, + q_history, + esnr_history, + write_delay_history, + read_osnr_delay_history, + read_transponder_delay_history, + bo_acquisition_delay_history, + total_delay_history, + ax_parameters, + param_constraints, + fiber_spans, + amp_type_id, + line, + sobol_num, + gpei_num, + metric_sign, + ssh_flag, + ssh_list, + user, + verbose, + ) + + return ( + best_parameters, + parameter_history, + reward_history, + osnr_history, + ber_history, + q_history, + esnr_history, + write_delay_history, + read_osnr_delay_history, + read_transponder_delay_history, + bo_acquisition_delay_history, + total_delay_history, + ) diff --git a/data/advance_amp.json b/data/advance_amp.json new file mode 100644 index 0000000..aeddecc --- /dev/null +++ b/data/advance_amp.json @@ -0,0 +1,304 @@ +{ + "nf_fit_coeff": [ + 0.000168241, + 0.0469961, + 0.0359549, + 5.82851 + ], + "f_min": 191350000000000.0, + "f_max": 196100000000000.0, + "nf_ripple": [ + -0.3110761646066259, + -0.3110761646066259, + -0.31110274831665313, + -0.31419329378173544, + -0.3172854168606314, + -0.32037911876162584, + -0.3233255190215882, + -0.31624321721895354, + -0.30915729645781326, + -0.30206775396360075, + -0.2949045115165272, + -0.26632156113294336, + -0.23772399031437283, + -0.20911178784023846, + -0.18048410390821285, + -0.14379944379052215, + -0.10709599992470213, + -0.07037375788020579, + -0.03372858157230583, + -0.015660302006048, + 0.0024172385953583004, + 0.020504047353947653, + 0.03860013139908377, + 0.05670549786742816, + 0.07482015390297145, + 0.0838762040768461, + 0.09284481475528361, + 0.1018180306253394, + 0.11079585523492333, + 0.1020395478432815, + 0.09310160456603413, + 0.08415906712621996, + 0.07521193198077789, + 0.0676340601339394, + 0.06005437964543287, + 0.052470799141237305, + 0.044883315610536455, + 0.037679759069084225, + 0.03047647598902483, + 0.02326948274513522, + 0.01605877647020772, + 0.021248462316134083, + 0.02657315875107553, + 0.03190060058247842, + 0.03723078993416436, + 0.04256372893215024, + 0.047899419704645264, + 0.03915515813685565, + 0.030289222542492025, + 0.021418708618354456, + 0.012573926129294415, + 0.006240488799898697, + -9.622162373026585e-05, + -0.006436207679519103, + -0.012779471908040341, + -0.02038153550619876, + -0.027999803010447587, + -0.035622012697103154, + -0.043236398934156144, + -0.04493583574805963, + -0.04663615264317309, + -0.048337350303318156, + -0.050039429413028365, + -0.051742390657545205, + -0.05342028484370278, + -0.05254242298580185, + -0.05166410580536087, + -0.05078533294804249, + -0.04990610405914272, + -0.05409792133358102, + -0.05832916277634124, + -0.06256260169582961, + -0.06660356886269536, + -0.04779792991567815, + -0.028982516728038848, + -0.010157321677553965, + 0.00861320615127981, + 0.01913736978785662, + 0.029667009055877668, + 0.04020212822983975, + 0.050742731588695494, + 0.061288823415841555, + 0.07184040799914815, + 0.1043252636301016, + 0.13687829834471027, + 0.1694483010211072, + 0.202035284929368, + 0.23624619427167134, + 0.27048596623174515, + 0.30474360397422756, + 0.3390191214858807, + 0.36358851509924695, + 0.38814205928193013, + 0.41270842850729195, + 0.4372876328262819, + 0.4372876328262819 + ], + "dgt": [ + 2.714526681131686, + 2.705443819238505, + 2.6947834587664494, + 2.6841217449620203, + 2.6681935771243177, + 2.6521732021128046, + 2.630396440815385, + 2.602860350286428, + 2.5696460593920065, + 2.5364027376452056, + 2.499446286796604, + 2.4587748041127506, + 2.414398437185221, + 2.3699990328716107, + 2.322373696229342, + 2.271520771371253, + 2.2174389328192197, + 2.16337565384239, + 2.1183028432496016, + 2.082225099873648, + 2.055100772005235, + 2.0279625371819305, + 2.0008103857988204, + 1.9736443063300082, + 1.9482128147680253, + 1.9245345552113182, + 1.9026104247588487, + 1.8806927939516411, + 1.862235672444246, + 1.847275503201129, + 1.835814081380705, + 1.824381436842932, + 1.8139629377087627, + 1.8045606557581335, + 1.7961751115773796, + 1.7877868031023945, + 1.7793941781790852, + 1.7709972329654864, + 1.7625959636196327, + 1.7541903672600494, + 1.7459181197626403, + 1.737780757913635, + 1.7297783508684146, + 1.7217732861435076, + 1.7137640932265894, + 1.7057507692361864, + 1.6918150918099673, + 1.6719047669939942, + 1.6460167077689267, + 1.6201194134191075, + 1.5986915141218316, + 1.5817353179379183, + 1.569199764184379, + 1.5566577309558969, + 1.545374152761467, + 1.5353620432989845, + 1.5266220576235803, + 1.5178910621476225, + 1.5097346239790443, + 1.502153039909686, + 1.495145456062699, + 1.488134243479226, + 1.48111939735681, + 1.474100442252211, + 1.4670307626366115, + 1.4599103316162523, + 1.45273959485914, + 1.445565137158368, + 1.4340878115214444, + 1.418273806730323, + 1.3981208704326855, + 1.3779439775587023, + 1.3598972673004606, + 1.3439818461440451, + 1.3301807335621048, + 1.316383926863083, + 1.3040618749785347, + 1.2932153453410835, + 1.2838336236692311, + 1.2744470198196236, + 1.2650555289898042, + 1.2556591482982988, + 1.2428104897182262, + 1.2264996957264114, + 1.2067249615595257, + 1.1869318618366975, + 1.1672278304018044, + 1.1476135933863398, + 1.1280891949729075, + 1.108555289615659, + 1.0895983485572227, + 1.0712204022764056, + 1.0534217504465226, + 1.0356155337864215, + 1.017807767853702, + 1.0 + ], + "gain_ripple": [ + 0.1359703369791596, + 0.11822862697916037, + 0.09542181697916163, + 0.06245819697916133, + 0.02602813697916062, + -0.0036199830208403228, + -0.018326963020840026, + -0.0246928330208398, + -0.016792253020838643, + -0.0028138630208403015, + 0.017572956979162058, + 0.038328296979159404, + 0.054956336979159914, + 0.0670723869791594, + 0.07091459697916136, + 0.07094413697916124, + 0.07114372697916238, + 0.07533675697916209, + 0.08731066697916035, + 0.10313984697916112, + 0.12276252697916235, + 0.14239527697916188, + 0.15945681697916214, + 0.1739275269791598, + 0.1767381569791624, + 0.17037189697916233, + 0.15216302697916007, + 0.13114358697916018, + 0.10802383697916085, + 0.08548825697916129, + 0.06916723697916183, + 0.05848224697916038, + 0.05447361697916264, + 0.05154489697916276, + 0.04946107697915991, + 0.04717897697916129, + 0.04551704697916037, + 0.04467697697916151, + 0.04072968697916224, + 0.03285456697916089, + 0.023488786979161347, + 0.01659282697915998, + 0.013321846979160057, + 0.011234826979162449, + 0.01030063697916006, + 0.00936596697916059, + 0.00874012697916271, + 0.00842583697916055, + 0.006965146979162284, + 0.0040435869791615175, + 0.0007104669791608842, + -0.0015763130208377163, + -0.006936193020838033, + -0.016475303020840215, + -0.028748483020837767, + -0.039618433020837784, + -0.051112303020840244, + -0.06468462302083822, + -0.07868024302083754, + -0.09101254302083817, + -0.10103437302083762, + -0.11041488302083735, + -0.11916081302083725, + -0.12789859302083784, + -0.1353792530208402, + -0.14160178302083892, + -0.1455411330208385, + -0.1484450830208388, + -0.14823350302084037, + -0.14591937302083835, + -0.1409032730208395, + -0.13525493302083902, + -0.1279646530208396, + -0.11963431302083904, + -0.11089282302084058, + -0.1027863830208382, + -0.09717347302083823, + -0.09343261302083761, + -0.0913487130208388, + -0.08906007302083907, + -0.0865687230208394, + -0.08407607302083875, + -0.07844600302084004, + -0.06968090302083851, + -0.05947139302083926, + -0.05095282302083959, + -0.042428283020839785, + -0.03218106302083967, + -0.01819858302084043, + -0.0021726530208390216, + 0.01393231697916164, + 0.028098946979159933, + 0.040326236979161934, + 0.05257029697916238, + 0.06479749697916048, + 0.07704745697916238 + ] +} diff --git a/data/default_edfa_config.json b/data/default_edfa_config.json new file mode 100644 index 0000000..f237b0b --- /dev/null +++ b/data/default_edfa_config.json @@ -0,0 +1,296 @@ +{ + "nf_ripple": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "gain_ripple": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "dgt": [ + 2.714526681131686, + 2.705443819238505, + 2.6947834587664494, + 2.6841217449620203, + 2.6681935771243177, + 2.6521732021128046, + 2.630396440815385, + 2.602860350286428, + 2.5696460593920065, + 2.5364027376452056, + 2.499446286796604, + 2.4587748041127506, + 2.414398437185221, + 2.3699990328716107, + 2.322373696229342, + 2.271520771371253, + 2.2174389328192197, + 2.16337565384239, + 2.1183028432496016, + 2.082225099873648, + 2.055100772005235, + 2.0279625371819305, + 2.0008103857988204, + 1.9736443063300082, + 1.9482128147680253, + 1.9245345552113182, + 1.9026104247588487, + 1.8806927939516411, + 1.862235672444246, + 1.847275503201129, + 1.835814081380705, + 1.824381436842932, + 1.8139629377087627, + 1.8045606557581335, + 1.7961751115773796, + 1.7877868031023945, + 1.7793941781790852, + 1.7709972329654864, + 1.7625959636196327, + 1.7541903672600494, + 1.7459181197626403, + 1.737780757913635, + 1.7297783508684146, + 1.7217732861435076, + 1.7137640932265894, + 1.7057507692361864, + 1.6918150918099673, + 1.6719047669939942, + 1.6460167077689267, + 1.6201194134191075, + 1.5986915141218316, + 1.5817353179379183, + 1.569199764184379, + 1.5566577309558969, + 1.545374152761467, + 1.5353620432989845, + 1.5266220576235803, + 1.5178910621476225, + 1.5097346239790443, + 1.502153039909686, + 1.495145456062699, + 1.488134243479226, + 1.48111939735681, + 1.474100442252211, + 1.4670307626366115, + 1.4599103316162523, + 1.45273959485914, + 1.445565137158368, + 1.4340878115214444, + 1.418273806730323, + 1.3981208704326855, + 1.3779439775587023, + 1.3598972673004606, + 1.3439818461440451, + 1.3301807335621048, + 1.316383926863083, + 1.3040618749785347, + 1.2932153453410835, + 1.2838336236692311, + 1.2744470198196236, + 1.2650555289898042, + 1.2556591482982988, + 1.2428104897182262, + 1.2264996957264114, + 1.2067249615595257, + 1.1869318618366975, + 1.1672278304018044, + 1.1476135933863398, + 1.1280891949729075, + 1.108555289615659, + 1.0895983485572227, + 1.0712204022764056, + 1.0534217504465226, + 1.0356155337864215, + 1.017807767853702, + 1.0 + ] +} diff --git a/data/eqpt_config.json b/data/eqpt_config.json new file mode 100644 index 0000000..5dbc0b1 --- /dev/null +++ b/data/eqpt_config.json @@ -0,0 +1,88 @@ +{ + "Edfa": [ + { + "type_variety": "high_detail_model_example", + "type_def": "advanced_model", + "gain_flatmax": 25, + "gain_min": 15, + "p_max": 21, + "advanced_config_from_json": "advance_amp.json", + "out_voa_auto": false, + "allowed_for_design": false + } + ], + "Fiber": [ + { + "type_variety": "SMF28", + "dispersion": 1.67e-05, + "gamma": 0.00124 + } + ], + "Span": [ + { + "power_mode": false, + "delta_power_range_db": [ + -2.0, + 3.0, + 0.5 + ], + "max_fiber_lineic_loss_for_raman": 0.25, + "target_extended_gain": 2.5, + "max_length": 150, + "length_units": "km", + "max_loss": 28, + "padding": 10, + "EOL": 0, + "con_in": 0, + "con_out": 0 + } + ], + "Roadm": [ + { + "target_pch_out_db": -20, + "add_drop_osnr": 38, + "restrictions": { + "preamp_variety_list": [], + "booster_variety_list": [] + } + } + ], + "SI": [ + { + "f_min": 193050000000000.0, + "baud_rate": 56800000000.0, + "f_max": 194325000000000.0, + "spacing": 75000000000.0, + "power_dbm": 2.0, + "power_range_db": [ + -5.0, + 1.0, + 0.5 + ], + "roll_off": 0.15, + "tx_osnr": 40, + "sys_margins": 2 + } + ], + "Transceiver": [ + { + "type_variety": "vendorA_trx-type1", + "frequency": { + "min": 193050000000000.0, + "max": 194325000000000.0 + }, + "mode": [ + { + "format": "mode 1", + "baud_rate": 56000000000.0, + "OSNR": 18, + "bit_rate": 300000000000.0, + "roll_off": 0.15, + "tx_osnr": 28, + "min_spacing": 75000000000.0, + "cost": 1 + } + ] + } + ] +} diff --git a/fcr_interface.py b/fcr_interface.py new file mode 100644 index 0000000..553322d --- /dev/null +++ b/fcr_interface.py @@ -0,0 +1,584 @@ +import copy +import datetime +import json +import logging +import re +import time +from collections import defaultdict + +import pyjq +from crypto.keychain_service.keychain import ttypes as keychain +from fbnet.command_runner.CommandRunner import ttypes as fcr_ttypes +from libfb.py.thrift_clients.fbnet_command_runner_thrift_client import ( + FBNetCommandRunnerThriftClient as Legacy_FcrClient, +) +from libfb.py.thrift_clients.keychain_thrift_client import KeychainClient +from scripts.neteng.optical.bow.ssh_interface import ( + ssh_read, + ssh_write, +) + +# define the class of line devices +class Line_Device(object): + def __init__( + self, + amplifiers, + wss, + wss_onoff, + proprietary_controller_info, + transponder_box, + transponder_fre, + fast_slow_version, + ): + self.amplifiers = amplifiers + self.wss = wss + self.wss_onoff = wss_onoff + self.proprietary_controller_info = proprietary_controller_info + self.transponder_box = transponder_box + self.transponder_fre = transponder_fre + self.fast_slow_version = fast_slow_version + + # system password for authetication + def get_secret(self, name, group): + req = keychain.GetSecretRequest( + name=name, + group=group, + ) + try: + return KeychainClient().getSecret(req) + except keychain.KeychainServiceException as ex: + print("Error retrieving secret:" + ex) + return False + + # activate user on device + def ssh_act_user(self, ssh): + print("Activating ssh user\n") + user = self.get_secret("TACACS_USERNAME", "NETENG_AUTOMATION").secret + pw = self.get_secret("TACACS_PASSWORD", "NETENG_AUTOMATION").secret + user_add_cmd = "XXXXXXXXXXX" + user + pw ## anonymized southbound command + ssh_write(ssh, user_add_cmd) + results = ssh_read(ssh, identify_sign="") + print("ssh_act_user results", results, "\n") + return user + + def ssh_cancel_user(self, user, ssh): + print("Canceling ssh user\n") + user_cancel_cmd = "XXXXXXXXXXX" + user ## anonymized southbound command + ssh_write(ssh, user_cancel_cmd) + results = ssh_read(ssh, identify_sign="") + print("ssh_cancel_user resutls", results, "\n") + + def tl1_ssh_runner(self, ssh, cmd, identify_sign): + ssh_write(ssh, cmd) + results = ssh_read(ssh, identify_sign=identify_sign) + micro_timestamp = datetime.datetime.now() + return results, micro_timestamp + + def tl1_device_bulkrunner(self, device_commands, read_or_write): + try: + user = self.get_secret("TACACS_USERNAME", "NETENG_AUTOMATION").secret + pw = self.get_secret("TACACS_PASSWORD", "NETENG_AUTOMATION").secret + if read_or_write == 1: + persistent_sign = "auto" + else: + persistent_sign = "auto" + + fcr_commands = { + fcr_ttypes.Device( + hostname=device, + username=user, + password=pw, + session_type=fcr_ttypes.SessionType.TL1, + session_data=fcr_ttypes.SessionData( + extra_options={ + "format": "json", + "use_persistent_connection": persistent_sign, + } + ), + ): device_commands[device] + for device in device_commands + } + with Legacy_FcrClient() as client: + res = client.bulk_run(fcr_commands) + timestamp = time.localtime() + return res, timestamp + + except Exception as ex: + print("User exception: {}".format(ex)) + timestamp = time.localtime() + return 1, timestamp + + # get the spectrum peaks of the channels on the testbed, vacant wavelength is 0 + def read_spectrum(self, wss, ssh_list, ssh_flag): + spectrum_spectrum = [] + noise_spectrum = [] + central_freq = [] + corrected_central_freq = [] + fine_grain_spectrum = [] + fine_grain_frequency = [] + bins = [] + + roadm_id = wss["roadm_id"] + chassis_id = wss["chassis_id"] + card_id = wss["card_id"] + port_id = wss["port_id"] + max_channel_read_num = wss["channel_num"] + grid_space = wss["grid"] + startchannel_freq = wss["start_freq"] + + if ssh_flag == 1: # use ssh one-time authentication + print("\033[1;31m\n** Use SSH direct connection for read_spectrum\033[0;0m") + command = "XXXXXXXXXXX" ## anonymized southbound command + + results, micro_timestamp = self.tl1_ssh_runner( + ssh_list[roadm_id], command, identify_sign="XXXXXXXXXXX" + ) + print("tl1_ssh_runner", results, "\n") + channel_id = [] + frequency = [] + power = [] + for s in results: + if s.startswith('"spectrum'): + channel_id.append(str(s[1:-1].split(",")[0])) + frequency.append(int(s[1:-1].split(",")[0].split("-")[-1])) + power.append(float(s[1:-1].split(",")[2])) + + else: # use FCR + print("\033[1;31m\n** Use FCR client connection for read_spectrum\033[0;0m") + command = "XXXXXXXXXXX" ## anonymized southbound command + + commands = defaultdict(list) + commands[roadm_id].append(command) + + rc = 1 + count = 0 + while rc == 1 and count < 10: # rc = 1 means exception + rc, timestamp = self.tl1_device_bulkrunner(commands, read_or_write=1) + count += 1 + + if rc == 1: + print("the FCR fails to fetch data\n") + channel_id = [] + frequency = [] + power = [] + else: + j_data = json.loads(rc[roadm_id][0].output) + vars = {} + jqscript = ".[][].fields | {keys: .[0][0], value: . [1][1]}" + results = pyjq.all(jqscript, j_data, vars=vars) + + channel_id = list(map(lambda x: str(x["keys"]), results)) + frequency = list(map(lambda x: int(x["keys"].split("-")[-1]), results)) + power = list(map(lambda x: float(x["value"]), results)) + + spectrum_reading = {channel_id[i]: power[i] for i in range(len(frequency))} + # print(spectrum_reading) + loc_match = "spectrum-" + chassis_id + "-" + card_id + "-" + port_id + # print(loc_match) + + step = 0 + channel_num = 0 + current_fre_bin = [] + current_power_bin = [] + + for item in range(len(spectrum_reading)): + if channel_id[item].startswith(loc_match): + fine_grain_spectrum.append(power[item]) + fine_grain_frequency.append(channel_id[item]) + if frequency[item] >= start_freq + grid_space * channel_num: + # print(frequency[item]) + current_fre_bin.append(frequency[item]) + current_power_bin.append(power[item]) + step = step + 1 + + if step == grid_space: + peak_power = max(current_power_bin) + loc = current_power_bin.index(peak_power) + noise = float( + ( + current_power_bin[0] + + current_power_bin[1] + + current_power_bin[-1] + + current_power_bin[-2] + ) + / 4 + ) + + if peak_power < -15: + pass + else: + spectrum_spectrum.append(peak_power) + central_freq.append(current_fre_bin[loc]) + noise_spectrum.append(noise) + current_fre_bin_copy = copy.deepcopy(current_fre_bin) + bins.append(current_fre_bin_copy) + + current_fre_bin.clear() + current_power_bin.clear() + channel_num = channel_num + 1 + step = 0 + + if channel_num == max_channel_read_num: + break + + ## if the detected central frequency is closest to one of the transponder frequency + for c in central_freq: + for x in self.transponder_fre: + if ( + abs(c - self.transponder_fre[x]) < 75000 + ): # frequency slot size is 75 GHz + corrected_central_freq.append(self.transponder_fre[x]) + break + + return ( + spectrum_spectrum, + noise_spectrum, + corrected_central_freq, + fine_grain_spectrum, + fine_grain_frequency, + bins, + ) + + def write_amp(self, parameters, ssh_list, ssh_flag): + write_amp_command = "XXXXXXXXXXX" ## anonymized southbound command + if ssh_flag: + print("\033[1;31m\n** Use SSH direct connection for write_amp\033[0;0m") + for amp in ssh_list: + print("write to", amp, "via ssh", ssh_list[amp]) + gain_command = ( + write_amp_command.format(amplifier_name=self.amplifiers[amp]) + + str(parameters[amp]) + + ";" + ) + rc, micro_timestamp = self.tl1_ssh_runner( + ssh_list[amp], gain_command, identify_sign="" + ) + print("tl1_ssh_runner", rc, "\n") + else: + print("\033[1;31m\n** Use FCR for write_amp\033[0;0m") + commands = defaultdict(list) + for amp in parameters: + gain_command = ( + write_amp_command.format(amplifier_name=self.amplifiers[amp]) + + str(parameters[amp]) + + ";" + ) + commands[amp].append(gain_command) + + rc = 1 + count = 0 + while rc == 1 and count < 10: # rc = 1 means exception + rc, timestamp = self.tl1_device_bulkrunner(commands, read_or_write=0) + count += 1 + + def read_amp(self, amplifiers): + query_amplifier_cmd = "XXXXXXXXXXX" ## anonymized southbound command + commands = defaultdict(list) + for amp in amplifiers: + commands[amp].append(query_amplifier_cmd.format(amp_name=amplifiers[amp])) + print(commands) + + results = 1 + count = 0 + while results == 1 and count < 10: # rc = 1 means exception + results, timestamp = self.tl1_device_bulkrunner(commands, read_or_write=1) + count += 1 + + gain_results = {} + if results == 1: + print("the FCR fails to fetch data") + else: + for device, command_results in results.items(): + for command_result in command_results: + if command_result.status != "success": + logging.error(f"{command_result.command} failed on {device}") + continue + gain = self.process_AMP(command_result) + gain_results[device] = gain + + return gain_results + + def process_AMP(self, rc): + j_data = json.loads(rc.output) + vars = {} + jqscript = ".[][].fields | {keys: .[0], value: .[2]}" + results = pyjq.all(jqscript, j_data, vars=vars) + gain = 0 + for i in range(len(results)): + if results[i]["value"]["AMPMODE"] == "GAINCLAMP": + gain = results[i]["value"]["GAIN"] + + return gain + + +# query transponder +def bulkrun_cli_command(device_commands, username, password): + fcr_commands = { + fcr_ttypes.Device( + hostname=device, + username=username, + password=password, + session_type=fcr_ttypes.SessionType.TL1, + session_data=fcr_ttypes.SessionData( + extra_options={ + "format": "json", + "use_persistent_connection": "auto", + } + ), + ): device_commands[device] + for device in device_commands + } + + with Legacy_FcrClient() as client: + res = client.bulk_run(fcr_commands) + return res + + +# bulk query +def bulk_query_cli(device_commands): + try: + username = self.get_secret("TACACS_USERNAME", "NETENG_AUTOMATION").secret + ppasswordw = self.get_secret("TACACS_PASSWORD", "NETENG_AUTOMATION").secret + response = bulkrun_cli_command( + device_commands=device_commands, + username=username, + password=password, + ) + timestamp = time.localtime() + return response, timestamp + + except Exception as ex: + print("exception: {}".format(ex)) + + +# clean the transponder collection bin +def clean_transponder(transponder_box): + ts_command_set = [ + "XXXXXXXXXXX" ## anonymized southbound command + ] + commands = defaultdict(list) + + ## construct commands for bulk run + for ts_box in transponder_box: + for transponder in transponder_box[ts_box]: + clear_command = ts_command_set[0] + transponder + commands[ts_box].append(clear_command) + + print("commands", commands) + rc, timestamp = bulk_query_cli(commands) + + +# query transponders adjacency +def get_transponder_adj(transponder_box, metric_bit): + ts_command_set = [ + "XXXXXXXXXXX" ## anonymized southbound command + ] + commands = defaultdict(list) + + ## construct commands for bulk run + for ts_box in transponder_box: + for transponder in transponder_box[ts_box]: + Modulation = "N/A" + Frequency = 0 + query_command = ts_command_set[0] + transponder ## query performance metric + commands[ts_box].append(query_command) + + print("get_transponder_adj commands", commands) + rc, timestamp = bulk_query_cli(commands) + + transponder_mod = {} + transponder_fre = {} + + for tx_box in transponder_box: + for tx in range(len(commands[tx_box])): + if metric_bit[0]: + regex_mod = r"Modulation Scheme" ## anonymized regex + matches_mod = re.finditer( + regex_mod, rc[tx_box][tx].output, re.MULTILINE + ) + for _, match in enumerate(matches_mod, start=1): + for groupNum in range(0, len(match.groups())): + groupNum = groupNum + 1 + Modulation = str(match.group(groupNum)) + + if metric_bit[1]: + regex_fre = r"Frequency \(GHz\)" ## anonymized regex + matches_fre = re.finditer( + regex_fre, rc[tx_box][tx].output, re.MULTILINE + ) + for _, match in enumerate(matches_fre, start=1): + for groupNum in range(0, len(match.groups())): + groupNum = groupNum + 1 + if match.group(groupNum) == "N/A": + Frequency = 999 + else: + Frequency = int(float(match.group(groupNum)) * 1000) + + transponder_mod[tx_box + "-" + transponder_box[tx_box][tx]] = Modulation + transponder_fre[tx_box + "-" + transponder_box[tx_box][tx]] = Frequency + + return transponder_mod, transponder_fre + + +## query transponders performance metric +def get_transponder_pm(line, ssh_list, metric_bit, ssh_flag): + clean_transponder(line.transponder_box) + time.sleep(5) + ts_command_set = [ + "XXXXXXXXXXX" ## anonymized southbound command + ] + if ssh_flag == 1: # use ssh one-time authentication + print( + "\033[1;31m\n** Use SSH direct connection for get_transponder_pm\033[0;0m" + ) + for ts_box in line.transponder_box: + for transponder in line.transponder_box[ts_box]: + BER = None + BERMax = None + Qfactor = None + QfactorMin = None + ESNR = None + ESNRmin = None + query_command = ( + ts_command_set[0] + + transponder + + " bin-type untimed" # untimed bin + ) ## query performance metric + + rc, micro_timestamp = line.tl1_ssh_runner( + ts_box, query_command, identify_sign="" + ) + print("tl1_ssh_runner", rc, "\n") + + else: # use FCR + print("\033[1;31m\n** Use FCR for get_transponder_pm\033[0;0m") + commands = defaultdict(list) + ## construct commands for bulk run + for ts_box in line.transponder_box: + for transponder in line.transponder_box[ts_box]: + BER = None + BERMax = None + Qfactor = None + QfactorMin = None + ESNR = None + ESNRmin = None + query_command = ( + ts_command_set[0] + + transponder + + " bin-type untimed" # untimed bin + ) ## query performance metric + commands[ts_box].append(query_command) + + print("commands", commands) + rc, timestamp = bulk_query_cli(commands) + + transponder_Q_set = {} + transponder_Qmin_set = {} + transponder_ber_set = {} + transponder_bermax_set = {} + transponder_esnr_set = {} + transponder_esnrmin_set = {} + + for tx_box in line.transponder_box: + for tx in range(len(commands[tx_box])): + BER = None + BERMax = None + Qfactor = None + QfactorMin = None + ESNR = None + ESNRmin = None + + if metric_bit[0]: + regex_q = r"Q-factor" ## anonymized regex + matches_q = re.finditer(regex_q, rc[tx_box][tx].output, re.MULTILINE) + for _, match in enumerate(matches_q, start=1): + for groupNum in range(0, len(match.groups())): + groupNum = groupNum + 1 + if match.group(groupNum) == "N/A": + Qfactor = None + else: + Qfactor = float(match.group(groupNum)) + + if metric_bit[1]: + regex_qmin = r"Q-factor Min" ## anonymized regex + matches_qmin = re.finditer( + regex_qmin, rc[tx_box][tx].output, re.MULTILINE + ) + for _, match in enumerate(matches_qmin, start=1): + for groupNum in range(0, len(match.groups())): + groupNum = groupNum + 1 + if match.group(groupNum) == "N/A": + QfactorMin = None + else: + QfactorMin = float(match.group(groupNum)) + + if metric_bit[2]: + regex_ber = r"Pre-FEC BER" ## anonymized regex + matches_ber = re.finditer( + regex_ber, rc[tx_box][tx].output, re.MULTILINE + ) + for _, match in enumerate(matches_ber, start=1): + for groupNum in range(0, len(match.groups())): + groupNum = groupNum + 1 + if match.group(groupNum) == "N/A": + BER = None + else: + BER = float(match.group(groupNum)) + + if metric_bit[3]: + regex_bermax = r"Pre-FEC BER Max" ## anonymized regex + matches_bermax = re.finditer( + regex_bermax, rc[tx_box][tx].output, re.MULTILINE + ) + for _, match in enumerate(matches_bermax, start=1): + for groupNum in range(0, len(match.groups())): + groupNum = groupNum + 1 + if match.group(groupNum) == "N/A": + BERMax = None + else: + BERMax = float(match.group(groupNum)) + + if metric_bit[4]: + regex_ESNR = r"ESNR Avg" ## anonymized regex + matches_bermax = re.finditer( + regex_ESNR, rc[tx_box][tx].output, re.MULTILINE + ) + for _, match in enumerate(matches_bermax, start=1): + for groupNum in range(0, len(match.groups())): + groupNum = groupNum + 1 + if match.group(groupNum) == "N/A": + ESNR = None + else: + ESNR = float(match.group(groupNum)) + + if metric_bit[5]: + regex_ESNRmin = r"ESNR Min" ## anonymized regex + matches_bermax = re.finditer( + regex_ESNRmin, rc[tx_box][tx].output, re.MULTILINE + ) + for _, match in enumerate(matches_bermax, start=1): + for groupNum in range(0, len(match.groups())): + groupNum = groupNum + 1 + if match.group(groupNum) == "N/A": + ESNRmin = None + else: + ESNRmin = float(match.group(groupNum)) + + frequency = line.transponder_fre[ + tx_box + "-" + line.transponder_box[tx_box][tx] + ] + transponder_Q_set[frequency] = Qfactor + transponder_Qmin_set[frequency] = QfactorMin + transponder_ber_set[frequency] = BER + transponder_bermax_set[frequency] = BERMax + transponder_esnr_set[frequency] = ESNR + transponder_esnrmin_set[frequency] = ESNRmin + + return ( + transponder_Q_set, + transponder_Qmin_set, + transponder_ber_set, + transponder_bermax_set, + transponder_esnr_set, + transponder_esnrmin_set, + ) diff --git a/input_parameters.py b/input_parameters.py new file mode 100644 index 0000000..9cf9d14 --- /dev/null +++ b/input_parameters.py @@ -0,0 +1,211 @@ +## network details +amplifiers_location = { + "amp1": "AMP-1", + "amp2": "AMP-1", + "amp3": "AMP-1", + "amp4": "AMP-1", + "amp5": "AMP-1", + "amp6": "AMP-1", + "amp7": "AMP-1", + "amp8": "AMP-1", +} +amp_typeid = { + "amp1": "high_detail_model_example", + "amp2": "high_detail_model_example", + "amp3": "high_detail_model_example", + "amp4": "high_detail_model_example", + "amp5": "high_detail_model_example", + "amp6": "high_detail_model_example", + "amp7": "high_detail_model_example", + "amp8": "high_detail_model_example", +} +## this is the WSS at the end of the rail where we can read spectrum +wss_location = { + "roadm_id": "roadm", + "chassis_id": "x", + "card_id": "x", + "port_id": "x", + "channel_num": 12, + "grid": 24, + "start_freq": 193012500, +} +## this is the proprietary_controller location which is responsible for managing this rail in the source ROADM site +proprietary_controller_info = { + "location_proprietary_controller": "x", + "proprietary_controller_id": "x", +} +transponder_box = { + "transponder1": ["1/1", "1/2", "2/1", "2/2"], + "transponder2": ["1/1", "1/2", "2/1", "2/2"], +} +wss_onoff = { + "node_id": "x", + "channel_name": [ + "CH1", # 194.25-194.325 + "CH2", # 194.1-194.175 + "CH3", # 193.95-194.025 + "CH4", # 193.8-193.875 + "CH5", # 193.65-193.725 + "CH6", # 193.35-193.425 + "CH7", # 193.2-193.275 + "CH8", # 193.05-193.125 + ], + "a_opqstatus": [ + "OPQ=YES", + "OPQ=YES", + "OPQ=YES", + "OPQ=YES", + "OPQ=YES", + "OPQ=YES", + "OPQ=YES", + "OPQ=NO", + ], + "z_opqstatus": [ + "OPQ=NO", + "OPQ=NO", + "OPQ=NO", + "OPQ=NO", + "OPQ=NO", + "OPQ=NO", + "OPQ=NO", + "OPQ=NO", + ], +} +channel = { + "name": [ + "channel0-1-1", + "channel0-1-2", + "channel0-1-3", + "channel0-1-4", + "channel0-1-5", + "channel0-1-6", + "channel0-1-7", + "channel0-1-8", + ], + "proprietary_controller_node": "x", + "a_status": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + ], + "z_status": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + ], +} + +fiber_spans = { + "amp1": { + "name": "amp1=amp2", + "length": 42, + "loss": 16.5, + }, + "amp2": { + "name": "amp2=amp3", + "length": 45, + "loss": 14.25, + }, + "amp3": { + "name": "amp3=amp4", + "length": 74, + "loss": 21.5, + }, + "amp4": { + "name": "amp4=amp5", + "length": 66, + "loss": 19.5, + }, + "amp5": { + "name": "amp5=anmp", + "length": 65, + "loss": 19.25, + }, + "amp6": { + "name": "amp6=amp7", + "length": 64, + "loss": 19 + }, + "amp7": { + "name": "amp7=amp8", + "length": 32, + "loss": 14 + }, +} +# constraints for gain of all amplifiers +param_constraints = [ + "amp1 + amp2 + amp3 + amp4 + amp5 + amp6 + amp7 + amp8 <= 134", + "amp1 + amp2 + amp3 + amp4 + amp5 + amp6 + amp7 + amp8 >= 128", +] + +ax_parameters = [ + { + "name": "amp1", + "type": "range", + "bounds": [12, 16], + "value_type": "float", + "digits": 1, + }, + { + "name": "amp2", + "type": "range", + "bounds": [13.5, 17.5], + "value_type": "float", + "digits": 1, + }, + { + "name": "amp3", + "type": "range", + "bounds": [16, 20], + "value_type": "float", + "digits": 1, + }, + { + "name": "amp4", + "type": "range", + "bounds": [18, 22], + "value_type": "float", + "digits": 1, + }, + { + "name": "amp5", + "type": "range", + "bounds": [17, 21], + "value_type": "float", + "digits": 1, + }, + { + "name": "amp6", + "type": "range", + "bounds": [13, 17], + "value_type": "float", + "digits": 1, + }, + { + "name": "amp7", + "type": "range", + "bounds": [14.5, 18.5], + "value_type": "float", + "digits": 1, + }, + { + "name": "amp8", + "type": "range", + "bounds": [12, 16], + "value_type": "float", + "digits": 1, + }, +] + +plots_location = "/home/zhizhenzhong/local/database/ofc21_plots/" +ampmonitor_location = "/home/zhizhenzhong/local/database/ampmonitor/" diff --git a/monitor.py b/monitor.py new file mode 100644 index 0000000..cdf8f96 --- /dev/null +++ b/monitor.py @@ -0,0 +1,178 @@ +import concurrent.futures +import sys +import time + +from bayesian_amp_control import get_line_osnr +from fcr_interface import ( + Line_Device, + get_transponder_pm, + get_transponder_adj, +) +from input_parameters import ( + amplifiers_location, + wss_location, + proprietary_controller_info, + transponder_box, + wss_onoff, +) + + +# polling the performance of amplifiers +def monitor_amp(iter_num, line): + timestamp = time.localtime() + pt_time = time.strftime("%Y.%m.%d.%H.%M.%S", timestamp) + + gain_time_series = [] + parameter_history = [] + + t = 0 + while t < iter_num: # repetitive measurement + gain_results = line.read_amp(line.amplifiers) + gain_results_float = [] + gain_results_key = [] + for x in gain_results: + gain_results_float.append(float(gain_results[x])) + gain_results_key.append(x) + gain_results_zip = dict(zip(gain_results_key, gain_results_float)) + parameter_history.append(gain_results_zip) + + timestamp = time.localtime() + pt_time = time.strftime("%Y.%m.%d.%H.%M.%S", timestamp) + gain_time_series.append(pt_time) + + print("\033[1;31m this is the " + str(t) + " amp query\033[0;0m") + print("amp gain_results_zip", gain_results_zip) + print("amp pt_time", pt_time) + t = t + 1 + + return parameter_history, gain_time_series + + +# polling the performance of transponders +def monitor_transponder(iter_num, line, ssh_list, metric_bit): + t = 0 + ber_history = [] + q_history = [] + esnr_history = [] + bermax_history = [] + qmin_history = [] + esnrmin_history = [] + tx_pm_time_series = [] + + while t < iter_num: # repetitive measurement + ( + transponder_Q_set, + transponder_QMin_set, + transponder_ber_set, + transponder_berMax_set, + transponder_esnr_set, + transponder_esnrMin_set, + ) = get_transponder_pm( + line, + ssh_list, + metric_bit, + ssh_flag=0, + ) + timestamp = time.localtime() + pt_time = time.strftime("%Y.%m.%d.%H.%M.%S", timestamp) + tx_pm_time_series.append(pt_time) + + print("\033[1;34m this is the " + str(t) + " transponder query\033[0;0m") + print("current Q", transponder_Q_set) + print("current BER", transponder_ber_set) + print("current ESNR", transponder_esnr_set) + print("current Q Min", transponder_QMin_set) + print("current BER Max", transponder_berMax_set) + print("current ESNR Min", transponder_esnrMin_set) + print("transponder pt_time", pt_time) + + ber_history.append(transponder_ber_set) + q_history.append(transponder_Q_set) + esnr_history.append(transponder_esnr_set) + bermax_history.append(transponder_berMax_set) + qmin_history.append(transponder_QMin_set) + esnrmin_history.append(transponder_esnrMin_set) + t = t + 1 + + return ( + ber_history, + q_history, + esnr_history, + bermax_history, + qmin_history, + esnrmin_history, + tx_pm_time_series, + ) + + +# polling the per-channel osnr +def monitor_wss(iter_num, line, ssh_list, ssh_flag): + t = 0 + osnr_history = [] + estimated_channel_osnr = [] + estimated_channel_power = [] + plot_flag = 0 + time_series = [] + + while t < iter_num: # repetitive measurement + current_osnr = get_line_osnr( + line, + ssh_list, + estimated_channel_osnr, + estimated_channel_power, + plot_flag, + ssh_flag, + ) + osnr_history.append(current_osnr) + timestamp = time.localtime() + pt_time = time.strftime("%Y.%m.%d.%H.%M.%S", timestamp) + time_series.append(pt_time) + print("\033[1;32m this is the " + str(t) + " WSS query\033[0;0m") + print(current_osnr) + print("OSNR pt_time", pt_time) + t = t + 1 + + return osnr_history, time_series + + +if __name__ == "__main__": + transponder_mod, transponder_fre = get_transponder_adj( + transponder_box, metric_bit=[1, 1] + ) + print("transponder_mod", transponder_mod) + print("transponder_fre", transponder_fre) + + # define line devices + line = Line_Device( + amplifiers_location, + wss_location, + wss_onoff, + proprietary_controller_info, + transponder_box, + transponder_fre, + fast_slow_version="fast", + ) + + iter_num = 30 + if line.fast_slow_version == "slow": + metric_bit = [1, 1, 1, 1, 1, 1] + else: + metric_bit = [1, 0, 1, 0, 1, 0] + + with concurrent.futures.ThreadPoolExecutor() as executor: + amp = executor.submit( + monitor_amp, + iter_num, + line, + ) + transponder = executor.submit( + monitor_transponder, + iter_num, + line, + metric_bit, + ) + + parameter_history, gain_time_series = amp.result() + ber_history, q_history, esnr_history, tx_pm_time_series = transponder.result() + + sys.exit(0) diff --git a/plotting.py b/plotting.py new file mode 100644 index 0000000..356aee2 --- /dev/null +++ b/plotting.py @@ -0,0 +1,407 @@ +import time +from collections import OrderedDict +import matplotlib.pyplot as plt +import numpy as np + + +def plot_channel( + channel_history, + colors, + pt_time, + sobol_index, + best_index, + plots_location, + metric_name, +): + wave_simple = [] + for w in channel_history[1]: + locals()[metric_name + str(w)] = [] + wave_simple.append(w) + color_dict = dict(zip(wave_simple, colors)) + + for t in range(len(channel_history)): + for q in channel_history[t]: + locals()[metric_name + str(q)].append(channel_history[t][q]) + x = range(len(channel_history)) + plt.figure(figsize=(10, 3)) + for w in channel_history[0]: + plt.scatter( + x[0], locals()[metric_name + str(w)][0], marker="o", color=color_dict[w] + ) + plt.plot( + x[1:-1], + locals()[metric_name + str(w)][1:-1], + label="channel_" + str(w), + linewidth=1, + marker="o", + color=color_dict[w], + ) + plt.scatter( + x[-1], + locals()[metric_name + str(w)][-1], + marker="*", + color=color_dict[w], + ) + plt.legend(loc="lower right") + plt.xlabel("BO trials") + plt.ylabel(metric_name) + if metric_name == "BER": + plt.yscale("log") + plt.xticks(np.arange(30)) + plt.axvline(x=sobol_index, color="gray", linestyle="--") + plt.axvline(x=best_index, color="gray", linestyle="--") + plt.tight_layout() + figure_name = metric_name + "_history_timestamp.png" + figure_name = figure_name.replace("timestamp", pt_time) + figure_name = plots_location + figure_name + plt.savefig(figure_name, dpi=100, bbox_inches="tight") + plt.show() + + + +def plot_spectrum( + line, + initial_spectrum, + initial_noise, + central_freq, + spectrum_detail, + freq_detail, + bins, + estimated_channel_power, +): + float_freq_detail = [float(x.split("-")[-1]) / 1000000 for x in freq_detail] + float_central_freq = [x / 1000000 for x in central_freq] + print("float_central_freq", float_central_freq) + print("estimated_channel_power", estimated_channel_power) + estimate_channel_power_list = [] + if len(estimated_channel_power) > 0: + for ch_thz in float_central_freq: + for est_ch_mhz in estimated_channel_power: + if abs(ch_thz * 1000000 - est_ch_mhz) < 75000: + estimate_channel_power_list.append( + estimated_channel_power[est_ch_mhz] + ) + break + location = ( + line.wss["roadm_id"] + + ":" + + str(line.wss["chassis_id"]) + + "-" + + str(line.wss["card_id"]) + + "-" + + str(line.wss["port_id"]) + ) + plt.figure(figsize=(15, 3)) + plt.plot( + float_freq_detail, + spectrum_detail, + linewidth=1, + label=location, + color="blue", + ) + for b in bins: + plt.axvline(x=float(b[0]) / 1000000, color="gray", linestyle="--", linewidth=1) + plt.axvline(x=float(b[-1]) / 1000000, color="red", linestyle="--", linewidth=1) + if len(float_central_freq) == len(initial_spectrum): + plt.scatter(float_central_freq, initial_spectrum, marker="*", color="blue") + if len(float_central_freq) == len(initial_noise): + plt.scatter(float_central_freq, initial_noise, marker="*", color="blue") + if len(estimated_channel_power) > 0: + plt.scatter( + float_central_freq, + estimate_channel_power_list, + marker="o", + color="green", + label="GNPy estimation", + ) + plt.ylabel("Power (dBm)") + plt.xlabel("Channel frequency (THz)") + plt.ylim(-30, 0) + plt.legend(loc="best", fontsize=14) + plt.show() + + +def line_plot( + plots_location, + amplifiers_location, + best_parameters, + best_index, + sobol_index, + parameter_history, + reward_history, + osnr_history, + ber_history, + q_history, + esnr_history, +): + x = range(len(parameter_history)) + timestamp = time.localtime() + pt_time = time.strftime("%Y.%m.%d.%H.%M.%S", timestamp) + colors = [ + "blue", + "orange", + "green", + "red", + "purple", + "brown", + "black", + "pink", + ] # color code for 8 items + + ## plot gain history + plt.figure(figsize=(10, 3)) + amps_simple = [] + sum_gain = [] + for a in amplifiers_location: + locals()["gain" + a] = [] + amps_simple.append(a) + for param in parameter_history: + current_gain = 0 + for a in param: + locals()["gain" + a].append(float(param[a])) + current_gain += float(param[a]) + sum_gain.append(current_gain) + + color_dict = dict(zip(amps_simple, colors)) + for a in amps_simple: + if len(x) == len(locals()["gain" + a]): + plt.scatter(x[0], locals()["gain" + a][0], color=color_dict[a], marker="o") + plt.plot( + x[1:], + locals()["gain" + a][1:], + label=a, + color=color_dict[a], + linewidth=1, + marker="o", + ) + plt.scatter( + x[best_index], + locals()["gain" + a][best_index], + color=color_dict[a], + marker="*", + ) + else: + print("plot vector size not equal") + print("x", x) + print(a, locals()["gain" + a]) + plt.legend(loc="lower right", bbox_to_anchor=(1, 0.5)) + plt.xlabel("BO trials") + plt.ylabel("amplifier parameter value") + plt.xticks(np.arange(30)) + plt.axvline(x=sobol_index, color="gray", linestyle="--") + plt.axvline(x=best_index, color="gray", linestyle="--") + plt.tight_layout() + figure_name = "param_history_timestamp.png" + figure_name = figure_name.replace("timestamp", pt_time) + figure_name = plots_location + figure_name + plt.savefig(figure_name, dpi=100, bbox_inches="tight") + + ## plot sum gain history + plt.figure(figsize=(10, 3)) + if len(x) == len(sum_gain): + plt.scatter(x[0], sum_gain[0], marker="o") + plt.plot( + x[1:], + sum_gain[1:], + linewidth=1, + marker="o", + ) + else: + print("plot vector size not equal") + print("x", x) + print("sum_gain", sum_gain) + plt.xlabel("BO trials") + plt.ylabel("SUM amplifier parameter value") + plt.xticks(np.arange(30)) + plt.axvline(x=sobol_index, color="gray", linestyle="--") + plt.axvline(x=best_index, color="gray", linestyle="--") + plt.tight_layout() + figure_name = "sumgain_history_timestamp.png" + figure_name = figure_name.replace("timestamp", pt_time) + figure_name = plots_location + figure_name + plt.savefig(figure_name, dpi=100, bbox_inches="tight") + plt.show() + + ## plot reward value history + plt.figure(figsize=(10, 3)) + rewards = [] + for xs in reward_history: + rewards.append(xs["bow"][0]) + x = range(len(rewards)) + if len(x) == len(rewards): + plt.scatter(x[0], rewards[0], marker="o") + plt.plot( + x[1:-1], + rewards[1:-1], + linewidth=1, + marker="o", + ) + plt.scatter(x[-1] + 1, rewards[-1], marker="*") + else: + print("plot vector size not equal") + print("x", x) + print("rewards", rewards) + plt.xlabel("BO trials") + plt.ylabel("reward") + plt.xticks(np.arange(30)) + plt.axvline(x=sobol_index, color="gray", linestyle="--") + plt.axvline(x=best_index, color="gray", linestyle="--") + plt.tight_layout() + figure_name = "reward_history_timestamp.png" + figure_name = figure_name.replace("timestamp", pt_time) + figure_name = plots_location + figure_name + plt.savefig(figure_name, dpi=100, bbox_inches="tight") + plt.show() + + # plot osnr of each wavelengths history + plot_channel( + osnr_history, + colors, + pt_time, + sobol_index, + best_index, + plots_location, + metric_name="osnr", + ) + + # plot BER of each wavelengths history + plot_channel( + ber_history, + colors, + pt_time, + sobol_index, + best_index, + plots_location, + metric_name="BER", + ) + + # plot Q of each wavelengths history + plot_channel( + q_history, + colors, + pt_time, + sobol_index, + best_index, + plots_location, + metric_name="Q", + ) + + # plot ESNR of each wavelengths history + plot_channel( + esnr_history, + colors, + pt_time, + sobol_index, + best_index, + plots_location, + metric_name="esnr", + ) + + +def bar_plot( + plots_location, + write_delay_history, + read_osnr_delay_history, + read_transponder_delay_history, + bo_acquisition_delay_history, + total_delay_history, +): + timestamp = time.localtime() + pt_time = time.strftime("%Y.%m.%d.%H.%M.%S", timestamp) + x = range(len(write_delay_history)) + plt.figure(figsize=(10, 3)) + plt.bar(x, write_delay_history, label="write_delay") + plt.bar(x, read_osnr_delay_history, bottom=write_delay_history, label="osnr_delay") + plt.bar( + x, + read_transponder_delay_history, + bottom=[ + write_delay_history[t] + read_osnr_delay_history[t] + for t in range(len(read_osnr_delay_history)) + ], + label="transponder_delay", + ) + plt.bar( + x, + bo_acquisition_delay_history, + bottom=[ + write_delay_history[t] + + read_osnr_delay_history[t] + + read_transponder_delay_history[t] + for t in range(len(read_osnr_delay_history)) + ], + label="BO acquisition delay", + ) + plt.xlabel("BO trials") + plt.ylabel("Operation delay (s)") + plt.xticks(np.arange(30)) + plt.legend(loc="lower right", bbox_to_anchor=(1, 0.5)) + plt.tight_layout() + figure_name = "decompose_delay_timestamp.png" + figure_name = figure_name.replace("timestamp", pt_time) + figure_name = plots_location + figure_name + plt.savefig(figure_name, dpi=100, bbox_inches="tight") + plt.show() + + plt.figure(figsize=(10, 3)) + plt.bar(x, total_delay_history, label="total delay") + plt.xlabel("BO trials") + plt.ylabel("Operation delay (s)") + plt.xticks(np.arange(30)) + plt.legend(loc="lower right", bbox_to_anchor=(1, 0.5)) + plt.tight_layout() + figure_name = "total_delay_timestamp.png" + figure_name = figure_name.replace("timestamp", pt_time) + figure_name = plots_location + figure_name + plt.savefig(figure_name, dpi=100, bbox_inches="tight") + plt.show() + + +def violin_plot( + plots_location, + parameter_all, + param_range, + pt_time, + xlabelname, + ylabelname, + figure_name, + log_y, +): + violin_list = [] + amp_set = [] + sorted_parameter_0 = OrderedDict( + sorted(parameter_all[0].items(), key=lambda t: t[0]) + ) + for n in sorted_parameter_0: + locals()["violin-" + str(n)] = [] + amp_set.append(n) + + for i in range(len(parameter_all)): + sorted_parameter = OrderedDict( + sorted(parameter_all[i].items(), key=lambda t: t[0]) + ) + for j in sorted_parameter: + locals()["violin-" + str(j)].append(sorted_parameter[j]) + + for n in sorted_parameter_0: + violin_list.append(locals()["violin-" + str(n)]) + + plt.figure() + plt.violinplot(violin_list, showmeans=True, showmedians=False) + dev_index = 0 + for dev in amp_set: + if dev in param_range: + dev_index += 1 + for bounds in param_range[dev]: + plt.scatter(dev_index, bounds, color="black", marker="o") + plt.xticks([y + 1 for y in range(len(violin_list))], amp_set) + plt.xticks(rotation=90) # Rotates X-Axis Ticks by 45-degrees + plt.xlabel(xlabelname) + plt.ylabel(ylabelname) + if log_y: + plt.yscale("log") + plt.tight_layout() + figure_name = figure_name.replace("timestamp", pt_time) + figure_name = plots_location + figure_name + plt.savefig(figure_name, dpi=100, bbox_inches="tight") + plt.show() diff --git a/run.py b/run.py new file mode 100644 index 0000000..117b4d5 --- /dev/null +++ b/run.py @@ -0,0 +1,200 @@ +import sys + +from bayesian_amp_control import ( + WavelengthReconfiguration, + Evaluate, + SafeCheck, +) +from input_parameters import ( + amplifiers_location, + wss_location, + proprietary_controller_info, + transponder_box, + wss_onoff, + fiber_spans, + plots_location, + param_constraints, + ax_parameters, + amp_type_id, +) +from plotting import line_plot, bar_plot, violin_plot +from fcr_interface import Line_Device, get_transponder_adj + + +def run( + plots_location, + amplifiers_location, + wss_location, + wss_onoff, + transponder_box, + proprietary_controller_info, + param_constraints, + ax_parameters, + sobol_num, + gpei_num, + metric_sign, + random_range, + fiber_spans, + amp_type_id, + fast_slow_version, + ssh_flag, + verbose, +): + print("\033[1;34m ==static query to know the setting of transponders==\033[0;0m") + + # specify transponders used for our experiment + transponder_mod, transponder_fre = get_transponder_adj( + transponder_box, metric_bit=[1, 1] + ) + print("transponder_mod", transponder_mod) + print("transponder_fre", transponder_fre) + + # specify line devices used for our experiment + line = Line_Device( + amplifiers_location, + wss_location, + wss_onoff, + proprietary_controller_info, + transponder_box, + transponder_fre, + fast_slow_version, + ) + print("line devices", line) + + ## run Bayesian Optimization loop + ( + best_parameters, + parameter_history, + reward_history, + osnr_history, + ber_history, + q_history, + esnr_history, + write_delay_history, + read_osnr_delay_history, + read_transponder_delay_history, + bo_acquisition_delay_history, + total_delay_history, + ) = WavelengthReconfiguration( + ax_parameters, + param_constraints, + fiber_spans, + amp_type_id, + line, + sobol_num=sobol_num, + gpei_num=gpei_num, + metric_sign=metric_sign, + random_range=random_range, + ssh_flag=ssh_flag, + verbose=verbose, + ) + + # print the parameter history during bO + print("osnr_history", osnr_history) + print("ber_history", ber_history) + print("q_history", q_history) + print("esnr_history", esnr_history) + + # figure out the BO index of the best parameter + best_index = 0 + for y in range(len(parameter_history)): + best_index_flag = 0 + for x in best_parameters: + if best_parameters[x] != parameter_history[y][x]: + best_index_flag = 1 + break + if best_index_flag == 0: + best_index = y + break + + # final process osnr_history + for k in range(len(osnr_history)): + osnr_keys = [] + if len(osnr_history[k]) == 0: + for x in osnr_keys: + osnr_history[k][x] = 0 + else: + osnr_keys = list(osnr_history[k].keys()) + + # plotting results for visualization + sobol_index = sobol_num + line_plot( + plots_location, + amplifiers_location, + best_parameters, + best_index, + sobol_index, + parameter_history, + reward_history, + osnr_history, + ber_history, + q_history, + esnr_history, + ) + bar_plot( + plots_location, + write_delay_history, + read_osnr_delay_history, + read_transponder_delay_history, + bo_acquisition_delay_history, + total_delay_history, + ) + + return ( + best_parameters, + best_index, + parameter_history, + reward_history, + osnr_history, + ber_history, + q_history, + esnr_history, + ) + + +if __name__ == "__main__": + print("\033[0;33m ================================================\n\033[0;0m") + print("\033[1;33m ###### ####### # #\033[0;0m") + print("\033[1;33m # # # # # # #\033[0;0m") + print("\033[1;33m ###### # # # # #\033[0;0m") + print("\033[1;33m # # # # # # #\033[0;0m") + print("\033[1;33m # # # # # # #\033[0;0m") + print("\033[1;33m ###### ####### ## ##\n\033[0;0m") + print("\033[1;34m BOW: Bayesian-Optimized Wavelengths\n\033[0;0m") + print("\033[0;34m [Paper]: Z. Zhong, M. Ghobadi, M. Balandat, S. Katti, A. Kazerouni, J. Leach, M. McKillop, Y. Zhang, BOW: First Real-World Demonstration of a Bayesian Optimization System for Wavelength Reconfiguration, OFC 2021 (Postdeadline Paper).\033[0;0m") + print("\033[0;34m [Website]: http://bow.csail.mit.edu\033[0;0m") + print("\033[0;34m [Code Contributor]: zhizhenz@mit.edu\033[0;0m") + print("\033[0;34m [Code Release Date]: August 4, 2021\033[0;0m") + print("\033[0;33m================================================\n\033[0;0m") + + + ( + best_parameters, + best_index, + parameter_history, + reward_history, + osnr_history, + ber_history, + q_history, + esnr_history, + ) = run( + plots_location, + amplifiers_location, + wss_location, + wss_onoff, + transponder_box, + proprietary_controller_info, + param_constraints, + ax_parameters, + sobol_num=10, + gpei_num=10, + metric_sign=0, + random_range=0, + fiber_spans=fiber_spans, + amp_type_id=amp_type_id, + fast_slow_version="fast", + ssh_flag=0, # 0 for FCR, 1 for direct SSH + verbose=True, + ) + + sys.exit() diff --git a/simulator.py b/simulator.py new file mode 100644 index 0000000..feebf3a --- /dev/null +++ b/simulator.py @@ -0,0 +1,711 @@ +# using GNPy v2.1 + +import getpass +import logging as lg +import time +import json +import networkx as nx + +import gnpy.core.ansi_escapes as ansi_escapes +import matplotlib.pyplot as plt +import numpy as np +import plotly.graph_objects as go +from gnpy.core.elements import Transceiver, Fiber, RamanFiber, Edfa +from gnpy.core.equipment import ( + load_equipment, + trx_mode_params, +) +from gnpy.core.exceptions import ( + ConfigurationError, + EquipmentConfigError, + NetworkTopologyError, +) +from gnpy.core.info import ( + SpectralInformation, + Channel, + Power, + Pref, +) +from gnpy.core.network import build_network, SimParams, network_from_json +from gnpy.core.request import Path_request, compute_constrained_path +from gnpy.core.utils import db2lin, lin2db +from plotly.subplots import make_subplots +# from scripts.neteng.optical.bow.input_parameters import fiber_spans + +logger = lg.getLogger(__name__) + +# Simulation parameters +tx_awg_mean_loss = 6.0 +tx_awg_rms_loss = 1.0 / 3.0 +patch_cable_rms_loss = 1.0 / 3.0 +tx_module_mean_output_power = -8.0 +tx_module_rms_output_power = 2.0 / 3.0 +zr_fbaud = 60e9 +zr_roll_off = 0.2 +zr_channel_spacing_ghz = 75.0 +crosstalk_penalty_base_rsnr = 26.0 - lin2db(zr_fbaud / 12.5e9) +# ROSNR penalty due to crosstalk, given for 1 neighbor, as a function of aggressor to signal power in dB +crosstalk_penalty = { + -10.0: 0.0, + -1.0: 0.1, + 0.0: 0.2, + 1.0: 0.3, + 2.0: 0.5, + 3.0: 0.8, + 4.0: 1.2, + 5.0: 1.7, + 6.0: 2.3, + 7.0: 3.0, +} +transceiver_min_ch_spacing = zr_channel_spacing_ghz * 1e9 + + +def format_gnpy_amp_network_element( + end_port_name: str, + amp_gain: float, + type_variety: str, + lat: float = 0.0, + lon: float = 0.0, +): + amp_element = { + "uid": f"Line Amp:{end_port_name}", + "type": "Edfa", + "type_variety": type_variety, + "operational": { + "gain_target": amp_gain, + "tilt_target": -0.5 + }, + "metadata": { + "location": { + "region": "", + "latitude": lat, + "longitude": lon, + } + }, + } + return amp_element + + +def format_gnpy_fiber_network_element( + uid: str, + length_km: float, + loss_db: float, + type_variety: str = "SMF28", + att_in: float = 1, + con_in: float = 1, + con_out: float = 1, + latitude: float = 0.0, + longitude: float = 0.0, + region: str = "LH", +): + loss_coef = (loss_db - att_in - con_in - con_out) / length_km + fiber_element = { + "uid": uid, + "type": "Fiber", + "type_variety": type_variety, + "params": { + "length": length_km, + "loss_coef": loss_coef, + "length_units": "km", + "att_in": att_in, + "con_in": con_in, + "con_out": con_out, + }, + "metadata": { + "location": { + "region": region, + "latitude": latitude, + "longitude": longitude, + }, + }, + } + return fiber_element + + +def format_gnpy_transponder_network_element( + uid: str, + type_variety: str = "vendorA_trx-type1", + lat: float = 0.0, + lon: float = 0.0, + city: str = "", + region: str = "", +): + return { + "uid": uid, + "type": "Transceiver", + "type_variety": type_variety, + "metadata": { + "city": city, + "region": region, + "latitutde": lat, + "longitude": lon, + }, + } + + +def Configure_GNPy_Network(amp_parameters, amp_type_id, fiber_spans): + gnpy_network = { + "network_name": "Southeast Asia Backbone", + "elements": [], + "connections": [], + } + src_transponder = format_gnpy_transponder_network_element("xpdr_start") + gnpy_network["elements"].append(src_transponder) + source = gnpy_network["elements"][0]["uid"] + last_node = source + + for amp in amp_parameters: + locals()["amp-" + amp] = format_gnpy_amp_network_element( + end_port_name=amp, + amp_gain=amp_parameters[amp], + type_variety=amp_type_id[amp], + ) + gnpy_network["elements"].append(locals()["amp-" + amp]) + connection = {"from_node": last_node, "to_node": locals()["amp-" + amp]["uid"]} + gnpy_network["connections"].append(connection) + last_node = locals()["amp-" + amp]["uid"] + + if amp in fiber_spans: + fiber_element = format_gnpy_fiber_network_element( + uid=fiber_spans[amp]["name"], + length_km=fiber_spans[amp]["length"], + loss_db=fiber_spans[amp]["loss"], + ) + gnpy_network["elements"].append(fiber_element) + connection = {"from_node": last_node, "to_node": fiber_element["uid"]} + gnpy_network["connections"].append(connection) + last_node = fiber_element["uid"] + + dst_transponder = format_gnpy_transponder_network_element("xpdr_end") + gnpy_network["elements"].append(dst_transponder) + dest = gnpy_network["elements"][-1]["uid"] + connection = {"from_node": last_node, "to_node": dest} + gnpy_network["connections"].append(connection) + + return gnpy_network, source, dest + + +def go_gnpy( + path, + equipment, + channel_frequencies, + crosstalk_penalty, + sim_params, + verbose, + greenfield: bool = False, + req_power=0.0, + debug_requests="", +): + nb_channel = len(channel_frequencies) + pref_ch_db = lin2db(req_power * 1e3) + + spans = [ + s.length for s in path if isinstance(s, RamanFiber) or isinstance(s, Fiber) + ] + print(f"Propagating through {len(spans)} fiber spans over {sum(spans)/1000:.0f} km") + + try: + p_start, p_stop, p_step = equipment["SI"]["default"].power_range_db + + except TypeError: + print( + "invalid power range definition in eqpt_config, should be power_range_db: [lower, upper, step]" + ) + + dp_db = 0.0 # Power offset, needs to include losses ahead of fiber span + power_optimization_incomplete = True + power_delta_range = [-5.0, 5.0] + + while power_optimization_incomplete: + req_power = db2lin(pref_ch_db + dp_db) * 1e-3 + pin = lin2db( + sum([db2lin(c["power"]) for c in channel_frequencies]) / nb_channel + ) + print( + f"Propagating with input power = {ansi_escapes.cyan}{lin2db(req_power*1e3):.2f} dBm{ansi_escapes.reset}:" + ) + pref = Pref(lin2db(req_power * 1e3), pin, lin2db(nb_channel)) + propagated_carriers = [] + next_osnr_penalty = 0.0 + for ix in range(nb_channel): + if ix < nb_channel - 1: + power_difference = ( + channel_frequencies[ix]["power"] + - channel_frequencies[ix + 1]["power"] + ) + osnr_penalty = np.interp( + -power_difference, + list(crosstalk_penalty.keys()), + list(crosstalk_penalty.values()), + ) + crosstalk_ase = ( + db2lin(-crosstalk_penalty_base_rsnr) + - db2lin(-crosstalk_penalty_base_rsnr - next_osnr_penalty) + + db2lin(-crosstalk_penalty_base_rsnr) + - db2lin(-crosstalk_penalty_base_rsnr - osnr_penalty) + ) + next_osnr_penalty = np.interp( + power_difference, + list(crosstalk_penalty.keys()), + list(crosstalk_penalty.values()), + ) + else: + crosstalk_ase = db2lin(-crosstalk_penalty_base_rsnr) - db2lin( + -crosstalk_penalty_base_rsnr - next_osnr_penalty + ) + pch = db2lin(channel_frequencies[ix]["power"]) * 1e-3 + if verbose: + print("pch", pch) + print("crosstalk_ase", crosstalk_ase) + new_ch = Channel( + ix + 1, + channel_frequencies[ix]["center_frequency"], + channel_frequencies[ix]["fbaud"], + channel_frequencies[ix]["roll_off"], + Power(pch, 0, pch * crosstalk_ase), + ) + propagated_carriers.append(new_ch) + si = SpectralInformation( + pref=pref, + carriers=propagated_carriers, + ) + + infos = {} + for el in path: + before_si = si + after_si = si = el(si) + + infos[el] = before_si, after_si + print("el", el) + + if "path_snr_accumulation" in debug_requests: + si_snr = [] + for si_carrier in si.carriers: + if si_carrier.power.ase: + si_snr.append( + si_carrier.power.signal + / (si_carrier.power.nli + si_carrier.power.ase) + ) + else: + si_snr.append(None) + + if greenfield: # Optimize launch power to balance ASE and NLI noise + if dp_db <= power_delta_range[0] or dp_db >= power_delta_range[1]: + power_optimization_incomplete = False + else: + ase_nli_power_imbal = ( + np.mean(path[-1].osnr_ase) - np.mean(path[-1].osnr_nli) + 3.0 + ) + dp_db -= 0.33 * ase_nli_power_imbal + dp_db = max([power_delta_range[0], dp_db]) + dp_db = min([power_delta_range[1], dp_db]) + power_optimization_incomplete = abs(ase_nli_power_imbal) > 0.05 + else: + break + + if "show_devices" in debug_requests: + print("=======") + print("GNPy simulation results") + for elem in path: + print(elem) + + return path, infos, [lin2db(req_power * 1e3), dp_db] + + +def go_est_snr( + amp_parameters, + amp_type_id, + fiber_spans, + equipment, + sim_params, + channel_frequencies, + crosstalk_penalty, + debug_requests, + verbose, + end_node="a", + greenfield=True, +): + max_launch_power_a_per_50ghz_dbm = -2.0 + + try: + network_json, source_uid, destination_uid = Configure_GNPy_Network( + amp_parameters, amp_type_id, fiber_spans + ) + except NetworkTopologyError: + return None, None, None, None + if network_json is None: + return None, None, None, None + + print("network_json", network_json) + + network = network_from_json(network_json, equipment) + print("network", network) + + plt.figure(figsize=(8,8)) + nx.draw(network, with_labels=True, node_color="white", edgecolors="blue", node_size=500) + plt.show() + + min_ch_spacing = transceiver_min_ch_spacing + launch_power_dbm = max_launch_power_a_per_50ghz_dbm + lin2db(min_ch_spacing / 50e9) + transceivers = {n.uid: n for n in network.nodes() if isinstance(n, Transceiver)} + source = transceivers[source_uid] + destination = transceivers[destination_uid] + + if not transceivers: + exit("Network has no transceivers!") + if len(transceivers) < 2: + exit("Network has only one transceiver!") + + # If no partial match or no source/destination provided pick random + if not source: + source = list(transceivers.values())[0] + del transceivers[source.uid] + + if not destination: + destination = list(transceivers.values())[0] + + params = {} + params["request_id"] = 0 + params["trx_type"] = "" + params["trx_mode"] = "" + params["source"] = source.uid + params["destination"] = destination.uid + params["nodes_list"] = [destination.uid] + params["loose_list"] = ["strict"] + params["format"] = "" + params["path_bandwidth"] = 0 + params["bidir"] = False + trx_params = trx_mode_params(equipment) + if launch_power_dbm: + trx_params["power"] = db2lin(float(launch_power_dbm)) * 1e-3 + params.update(trx_params) + req = Path_request(**params) + nb_channel = len(channel_frequencies) + pref_ch_db = lin2db(req.power * 1e3) # reference channel power / span (SL=20dB) + pref_total_db = pref_ch_db + lin2db(nb_channel) + build_network(network, equipment, pref_ch_db, pref_total_db) + path = compute_constrained_path(network, req) + print("path: ", path) + + if verbose: + print("=======") + print("before GNPy simulation") + for elem in path: + print(elem) + + path, infos, launch_power = go_gnpy( + path, + equipment, + channel_frequencies, + crosstalk_penalty, + sim_params, + verbose, + greenfield=greenfield, + req_power=req.power, + debug_requests=debug_requests, + ) + # save_network(filename, network) + + if "show_channels" in debug_requests: + print("\nThe total SNR per channel (in signal BW) at the end of the line is:") + print( + "{:>5}{:>15}{:>14}{:>12}{:>12}{:>12}".format( + "Ch. #", + "Ch freq (THz)", + "Ch Pwr (dBm)", + "OSNR ASE", + "SNR NLI", + "SNR total", + ) + ) + for final_carrier, ch_osnr, ch_snr_nl, ch_snr in zip( + infos[path[-1]][1].carriers, + path[-1].osnr_ase, + path[-1].osnr_nli, + path[-1].snr, + ): + ch_freq = final_carrier.frequency * 1e-12 + ch_power = lin2db(final_carrier.power.signal * 1e3) + print( + "{:5}{:15.4f}{:14.2f}{:12.2f}{:12.2f}{:12.2f}".format( + final_carrier.channel_number, + round(ch_freq, 4), + round(ch_power, 2), + round(ch_osnr, 2), + round(ch_snr_nl, 2), + round(ch_snr, 2), + ) + ) + + if "do_plot" in debug_requests: + fig = plt.figure(figsize=(14, 6)) + ax = fig.add_subplot(111) + xpl = [0.0] + ypl = [0.0] + dd = 0.0 + ttx = [] + tty = [] + tt = [] + for ix in range(len(path)): + if type(path[ix]) is Transceiver: + ttx.append(xpl[-1]) + tty.append(ypl[-1]) + tt.append(path[ix].uid) + if type(path[ix]) is Edfa: + xpl.append(xpl[-1] + dd) + ypl.append(path[ix].pin_db) + xpl.append(xpl[-1]) + ypl.append(path[ix].pout_db) + ttx.append(xpl[-1]) + tty.append(ypl[-1]) + tt1 = path[ix].uid.split(":") + tt.append(tt1[1]) + dd = 0.0 + if type(path[ix]) is Fiber: + dd += path[ix].length * 1e-3 + ax.plot(xpl, ypl) + for ix in range(len(tt)): + ax.text(ttx[ix], tty[ix], tt[ix], rotation=90) + ax.set(xlabel="Route distance [km]", ylabel="Aggregate Power [dBm]") + return path, infos, launch_power + + +def run_simulator( + amp_parameters, + amp_type_id, + fiber_spans, + left_freq_mhz, + right_freq_mhz, + verbose, +): + USER = getpass.getuser() + channel_osnr = {} + channel_power = {} + file_path = f"./data/" + equipment_file = file_path + "eqpt_config.json" + + sim_params = SimParams( + { + "raman_computed_channels": [1, 18, 37, 56, 75], + "raman_parameters": { + "flag_raman": True, + "space_resolution": 10e3, + "tolerance": 1e-8, + }, + "nli_parameters": { + "nli_method_name": "ggn_spectrally_separated", + "wdm_grid_size": zr_channel_spacing_ghz * 1e9, + "dispersion_tolerance": 1, + "phase_shift_tollerance": 0.1, + }, + } + ) + + try: + equipment = load_equipment(equipment_file) + except EquipmentConfigError as e: + print( + f"{ansi_escapes.red}Configuration error in the equipment library:{ansi_escapes.reset} {e}" + ) + exit(1) + except NetworkTopologyError as e: + print(f"{ansi_escapes.red}Invalid network definition:{ansi_escapes.reset} {e}") + exit(1) + except ConfigurationError as e: + print(f"{ansi_escapes.red}Configuration error:{ansi_escapes.reset} {e}") + exit(1) + + channel_frequencies = [] + ## uniform spectrum + # channel_num = int((right_freq_mhz - left_freq_mhz) / 75000) - 1 ## channel spacing is 75 GHz + # chf = np.linspace( + # left_freq_mhz + zr_channel_spacing_ghz * 1e3 / 2, + # right_freq_mhz - zr_channel_spacing_ghz * 1e3 / 2, + # channel_num, + # ) + ## other spectrum + channel_num = 8 + chf = [ + 193087500, # CH8 + 193237500, # CH7 + 193387500, # CH6 + 193687500, # CH5 + 193837500, # CH4 + 193987500, # CH3 + 194137500, # CH2 + 194287500, # CH1 + ] + + for ch_ix in range(len(chf)): + tx_pwr = ( + tx_module_mean_output_power + - tx_awg_mean_loss + + float(np.random.randn(1)) * tx_awg_rms_loss + - abs(float(np.random.randn(1)) * patch_cable_rms_loss) + + float(np.random.randn(1)) * tx_module_rms_output_power + ) + channel_frequencies.append( + { + "center_frequency": chf[ch_ix], + "fbaud": zr_fbaud, + "roll_off": zr_roll_off, + "width": zr_channel_spacing_ghz * 1e9, + "power": tx_pwr, + } + ) + t0 = time.perf_counter() + debug_requests = [ + "do_plot", + "show_channels", + "show_devices", + ] + path, infos, launch_power = go_est_snr( + amp_parameters, + amp_type_id, + fiber_spans, + equipment, + sim_params, + channel_frequencies, + crosstalk_penalty, + debug_requests=debug_requests, + verbose=verbose, + end_node="a", + greenfield=False, + ) + t1 = time.perf_counter() + print(f"Elapsed time for GNPy calc is {t1-t0:.2f} seconds") + + snr_required = 27 - lin2db(zr_fbaud / 12.5e9) + if verbose: + print("\nThe total SNR per channel (in signal BW) at the end of the line is:") + print( + "{:>5}{:>15}{:>14}{:>12}{:>12}{:>12}".format( + "Ch. #", + "Ch freq (MHz)", + "Ch Pwr (dBm)", + "OSNR ASE", + "SNR NLI", + "SNR total", + ) + ) + + for final_carrier, ch_osnr, ch_snr_nl, ch_snr in zip( + infos[path[-1]][1].carriers, path[-1].osnr_ase, path[-1].osnr_nli, path[-1].snr + ): + ch_freq = int(final_carrier.frequency) # MHz + ch_power = lin2db(final_carrier.power.signal * 1e3) + channel_osnr[ch_freq] = round(ch_snr, 2) + channel_power[ch_freq] = round(ch_power, 2) + if verbose: + if ch_osnr < snr_required: + print( + f"{final_carrier.channel_number:5}{ch_freq:15.4f}{ch_power:14.2f}{ch_osnr:12.2f}" + f"{ch_snr_nl:12.2f}{ansi_escapes.red}{ch_snr:12.2f}{ansi_escapes.reset}" + ) + else: + print( + "{:5}{:15.4f}{:14.2f}{:12.2f}{:12.2f}{:12.2f}".format( + final_carrier.channel_number, + ch_freq, + round(ch_power, 2), + round(ch_osnr, 2), + round(ch_snr_nl, 2), + round(ch_snr, 2), + ) + ) + + ch_freq = [] + ch_power = [c["power"] for c in channel_frequencies] + snr = [] + for final_carrier, _ch_osnr, _ch_snr_nl, ch_snr in zip( + infos[path[-1]][1].carriers, path[-1].osnr_ase, path[-1].osnr_nli, path[-1].snr + ): + ch_freq.append(final_carrier.frequency * 1e-12) + snr.append(ch_snr) + + return channel_osnr, channel_power + + +if __name__ == "__main__": + ## example data to test the GNPy simulator + parameters = { + "amp1": 14.2, + "amp2": 16.6, + "amp3": 18.8, + "amp4": 18.2, + "amp5": 16.8, + "amp6": 15.8, + "amp7": 15.0, + "amp8": 14.5, + } + amp_type_id = { + "amp1": "high_detail_model_example", + "amp2": "high_detail_model_example", + "amp3": "high_detail_model_example", + "amp4": "high_detail_model_example", + "amp5": "high_detail_model_example", + "amp6": "high_detail_model_example", + "amp7": "high_detail_model_example", + "amp8": "high_detail_model_example", + } + fiber_spans = { + "amp1": { + "name": "amp1=amp2", + "length": 42, + "loss": 16.5, + }, + "amp2": { + "name": "amp2=amp3", + "length": 45, + "loss": 14.25, + }, + "amp3": { + "name": "amp3=amp4", + "length": 74, + "loss": 21.5, + }, + "amp4": { + "name": "amp4=amp5", + "length": 66, + "loss": 19.5, + }, + "amp5": { + "name": "amp5=amp6", + "length": 65, + "loss": 19.25, + }, + "amp6": { + "name": "amp6=amp7", + "length": 64, + "loss": 19 + }, + "amp7": { + "name": "amp7=amp8", + "length": 32, + "loss": 14 + }, + } + left_freq = 193050000 + right_freq = 194325000 + + estimated_channel_osnr = {} + for _ in range(10): + estimated_channel_osnr_current, estimated_channel_power_current = run_simulator( + parameters, + amp_type_id, + fiber_spans, + left_freq - 37500, + right_freq + 37500, + verbose=True, + ) # 4800 GHz C-band spectrum, @75 GHz spacing + if len(estimated_channel_osnr) > 0: + for ch in estimated_channel_osnr_current: + estimated_channel_osnr[ch] += estimated_channel_osnr_current[ch] + else: + for ch in estimated_channel_osnr_current: + estimated_channel_osnr[ch] = estimated_channel_osnr_current[ch] + print(estimated_channel_osnr_current) + for ch in estimated_channel_osnr: + estimated_channel_osnr[ch] = estimated_channel_osnr[ch]/10 + + print("estimated_channel_osnr", estimated_channel_osnr) \ No newline at end of file diff --git a/ssh_interface.py b/ssh_interface.py new file mode 100644 index 0000000..2645fcf --- /dev/null +++ b/ssh_interface.py @@ -0,0 +1,53 @@ +import subprocess + + +def ssh_setup(host_name): + host_name = host_name + "-1" + ssh = subprocess.Popen( + ["ssh", "-i .ssh/id_rsa", host_name], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + bufsize=0, + ) + print("Setting direct ssh to", host_name) + return ssh + + +def ssh_write(ssh, cmd): + print("* ssh write", cmd) + try: + ssh.stdin.write(cmd) + except Exception as ex: + print("Exceptions", ex) + ssh_close(ssh) + + +def ssh_read(ssh, identify_sign): + print("* ssh read begins") + results = [] + last_identify = "NA" + for line in ssh.stdout: + if identify_sign in line.strip(): + results.append(line.strip()) + last_identify = identify_sign + else: + last_identify = "NA" + + if line.strip() == ";" and last_identify == identify_sign: + break + if len(results): + if "196125000" in results[-1]: # test spectrum end frequency + break + + print("* ssh read ends") + return results + + +def ssh_close(ssh): + ssh.stdin.close() + ssh.stdout.close() + ssh.stderr.close() + ssh.kill() + print("close ssh connection")