From be43d9be1271923c48e6cf92da235a4a894d6cd2 Mon Sep 17 00:00:00 2001 From: Xincun Li Date: Tue, 30 Apr 2024 15:41:37 -0700 Subject: [PATCH 01/44] Add Checkpoint and Rollback for Multi ASIC. --- config/main.py | 45 +++++++-- generic_config_updater/generic_updater.py | 13 ++- tests/config_test.py | 118 +++++++++++++++++++++- 3 files changed, 164 insertions(+), 12 deletions(-) diff --git a/config/main.py b/config/main.py index 8f3b7245bd..61ffc1a03c 100644 --- a/config/main.py +++ b/config/main.py @@ -1168,7 +1168,7 @@ def apply_patch_for_scope(scope_changes, results, config_format, verbose, dry_ru log.log_notice(f"'apply-patch' executed successfully for {scope_for_log} by {changes}") except Exception as e: results[scope_for_log] = {"success": False, "message": str(e)} - log.log_error(f"'apply-patch' executed failed for {scope_for_log} by {changes} due to {str(e)}") + log.log_warning(f"'apply-patch' executed failed for {scope_for_log} by {changes} due to {str(e)}") # This is our main entrypoint - the main 'config' command @@ -1397,7 +1397,8 @@ def apply_patch(ctx, patch_file_path, format, dry_run, ignore_non_yang_tables, i # Empty case to force validate YANG model. if not changes_by_scope: asic_list = [multi_asic.DEFAULT_NAMESPACE] - asic_list.extend(multi_asic.get_namespace_list()) + if multi_asic.is_multi_asic(): + asic_list.extend(multi_asic.get_namespace_list()) for asic in asic_list: changes_by_scope[asic] = [] @@ -1420,6 +1421,10 @@ def apply_patch(ctx, patch_file_path, format, dry_run, ignore_non_yang_tables, i @config.command() @click.argument('target-file-path', type=str, required=True) +@click.option('-s', '--scope', + help='Specify the namespace for Multi-ASIC environments. For single ASIC environments or Multi-ASIC localhost, specifying the namespace is not required.', + required=True if multi_asic.is_multi_asic() else False, + type=click.Choice(multi_asic.get_namespace_list() + ['localhost'])) @click.option('-f', '--format', type=click.Choice([e.name for e in ConfigFormat]), default=ConfigFormat.CONFIGDB.name, help='format of target config is either ConfigDb(ABNF) or SonicYang', @@ -1429,14 +1434,18 @@ def apply_patch(ctx, patch_file_path, format, dry_run, ignore_non_yang_tables, i @click.option('-i', '--ignore-path', multiple=True, help='ignore validation for config specified by given path which is a JsonPointer', hidden=True) @click.option('-v', '--verbose', is_flag=True, default=False, help='print additional details of what the operation is doing') @click.pass_context -def replace(ctx, target_file_path, format, dry_run, ignore_non_yang_tables, ignore_path, verbose): +def replace(ctx, target_file_path, scope, format, dry_run, ignore_non_yang_tables, ignore_path, verbose): """Replace the whole config with the specified config. The config is replaced with minimum disruption e.g. if ACL config is different between current and target config only ACL config is updated, and other config/services such as DHCP will not be affected. **WARNING** The target config file should be the whole config, not just the part intended to be updated. - : Path to the target file on the file-system.""" + : Path to the target file on the file-system. + + If the device is a Multi-ASIC environment, please give the namespace of specific ASIC, for instance: + localhost, asic0, asic1, ... + """ try: print_dry_run_message(dry_run) @@ -1445,8 +1454,13 @@ def replace(ctx, target_file_path, format, dry_run, ignore_non_yang_tables, igno target_config = json.loads(target_config_as_text) config_format = ConfigFormat[format.upper()] - - GenericUpdater().replace(target_config, config_format, verbose, dry_run, ignore_non_yang_tables, ignore_path) + if multi_asic.is_multi_asic(): + if scope not in (multi_asic.get_namespace_list() + ["localhost"]): + raise Exception(f"Failed to replace config due to wrong namespace:{scope}") + scope = scope if scope != "localhost" else multi_asic.DEFAULT_NAMESPACE + else: + scope = multi_asic.DEFAULT_NAMESPACE + GenericUpdater(namespace=scope).replace(target_config, config_format, verbose, dry_run, ignore_non_yang_tables, ignore_path) click.secho("Config replaced successfully.", fg="cyan", underline=True) except Exception as ex: @@ -1468,8 +1482,11 @@ def rollback(ctx, checkpoint_name, dry_run, ignore_non_yang_tables, ignore_path, : The checkpoint name, use `config list-checkpoints` command to see available checkpoints.""" try: print_dry_run_message(dry_run) - - GenericUpdater().rollback(checkpoint_name, verbose, dry_run, ignore_non_yang_tables, ignore_path) + asic_list = [multi_asic.DEFAULT_NAMESPACE] + if multi_asic.is_multi_asic(): + asic_list.extend(multi_asic.get_namespace_list()) + for asic in asic_list: + GenericUpdater(namespace=asic).rollback(checkpoint_name, verbose, dry_run, ignore_non_yang_tables, ignore_path) click.secho("Config rolled back successfully.", fg="cyan", underline=True) except Exception as ex: @@ -1485,7 +1502,11 @@ def checkpoint(ctx, checkpoint_name, verbose): : The checkpoint name, use `config list-checkpoints` command to see available checkpoints.""" try: - GenericUpdater().checkpoint(checkpoint_name, verbose) + asic_list = [multi_asic.DEFAULT_NAMESPACE] + if multi_asic.is_multi_asic(): + asic_list.extend(multi_asic.get_namespace_list()) + for asic in asic_list: + GenericUpdater(namespace=asic).checkpoint(checkpoint_name, verbose) click.secho("Checkpoint created successfully.", fg="cyan", underline=True) except Exception as ex: @@ -1501,7 +1522,11 @@ def delete_checkpoint(ctx, checkpoint_name, verbose): : The checkpoint name, use `config list-checkpoints` command to see available checkpoints.""" try: - GenericUpdater().delete_checkpoint(checkpoint_name, verbose) + asic_list = [multi_asic.DEFAULT_NAMESPACE] + if multi_asic.is_multi_asic(): + asic_list.extend(multi_asic.get_namespace_list()) + for asic in asic_list: + GenericUpdater(namespace=asic).delete_checkpoint(checkpoint_name, verbose) click.secho("Checkpoint deleted successfully.", fg="cyan", underline=True) except Exception as ex: diff --git a/generic_config_updater/generic_updater.py b/generic_config_updater/generic_updater.py index b75939749c..20fad576ca 100644 --- a/generic_config_updater/generic_updater.py +++ b/generic_config_updater/generic_updater.py @@ -84,7 +84,7 @@ def apply(self, patch, sort=True): self.config_wrapper.validate_field_operation(old_config, target_config) # Validate target config does not have empty tables since they do not show up in ConfigDb - self.logger.log_notice(f"{scope}: alidating target config does not have empty tables, " \ + self.logger.log_notice(f"{scope}: validating target config does not have empty tables, " \ "since they do not show up in ConfigDb.") empty_tables = self.config_wrapper.get_empty_tables(target_config) if empty_tables: # if there are empty tables @@ -169,6 +169,7 @@ def __init__(self, self.config_wrapper = config_wrapper if config_wrapper is not None else ConfigWrapper(namespace=self.namespace) def rollback(self, checkpoint_name): + checkpoint_name = checkpoint_name + self.namespace self.logger.log_notice("Config rollbacking starting.") self.logger.log_notice(f"Checkpoint name: {checkpoint_name}.") @@ -185,6 +186,7 @@ def rollback(self, checkpoint_name): self.logger.log_notice("Config rollbacking completed.") def checkpoint(self, checkpoint_name): + checkpoint_name = checkpoint_name + self.namespace self.logger.log_notice("Config checkpoint starting.") self.logger.log_notice(f"Checkpoint name: {checkpoint_name}.") @@ -223,6 +225,7 @@ def list_checkpoints(self): return checkpoint_names def delete_checkpoint(self, checkpoint_name): + checkpoint_name = checkpoint_name + self.namespace self.logger.log_notice("Deleting checkpoint starting.") self.logger.log_notice(f"Checkpoint name: {checkpoint_name}.") @@ -291,15 +294,18 @@ def replace(self, target_config): self.decorated_config_replacer.replace(target_config) def rollback(self, checkpoint_name): + checkpoint_name = checkpoint_name + self.namespace self.decorated_config_rollbacker.rollback(checkpoint_name) def checkpoint(self, checkpoint_name): + checkpoint_name = checkpoint_name + self.namespace self.decorated_config_rollbacker.checkpoint(checkpoint_name) def list_checkpoints(self): return self.decorated_config_rollbacker.list_checkpoints() def delete_checkpoint(self, checkpoint_name): + checkpoint_name = checkpoint_name + self.namespace self.decorated_config_rollbacker.delete_checkpoint(checkpoint_name) @@ -338,9 +344,11 @@ def replace(self, target_config): self.execute_write_action(Decorator.replace, self, target_config) def rollback(self, checkpoint_name): + checkpoint_name = checkpoint_name + self.namespace self.execute_write_action(Decorator.rollback, self, checkpoint_name) def checkpoint(self, checkpoint_name): + checkpoint_name = checkpoint_name + self.namespace self.execute_write_action(Decorator.checkpoint, self, checkpoint_name) def execute_write_action(self, action, *args): @@ -475,14 +483,17 @@ def replace(self, target_config, config_format, verbose, dry_run, ignore_non_yan config_replacer.replace(target_config) def rollback(self, checkpoint_name, verbose, dry_run, ignore_non_yang_tables, ignore_paths): + checkpoint_name = checkpoint_name + self.namespace config_rollbacker = self.generic_update_factory.create_config_rollbacker(verbose, dry_run, ignore_non_yang_tables, ignore_paths) config_rollbacker.rollback(checkpoint_name) def checkpoint(self, checkpoint_name, verbose): + checkpoint_name = checkpoint_name + self.namespace config_rollbacker = self.generic_update_factory.create_config_rollbacker(verbose) config_rollbacker.checkpoint(checkpoint_name) def delete_checkpoint(self, checkpoint_name, verbose): + checkpoint_name = checkpoint_name + self.namespace config_rollbacker = self.generic_update_factory.create_config_rollbacker(verbose) config_rollbacker.delete_checkpoint(checkpoint_name) diff --git a/tests/config_test.py b/tests/config_test.py index 1054a52a33..05a23a336f 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -2651,6 +2651,7 @@ def setUp(self): self.runner = CliRunner() self.patch_file_path = 'path/to/patch.json' + self.replace_file_path = 'path/to/replace.json' self.patch_content = [ { "op": "add", @@ -2731,6 +2732,121 @@ def test_apply_patch_dryrun_multiasic(self): # Ensure ConfigDBConnector was never instantiated or called mock_config_db_connector.assert_not_called() + + def test_repalce_multiasic(self): + # Mock open to simulate file reading + mock_replace_content = "[]" + with patch('builtins.open', mock_open(read_data=json.dumps(mock_replace_content)), create=True) as mocked_open: + # Mock GenericUpdater to avoid actual patch application + with patch('config.main.GenericUpdater') as mock_generic_updater: + mock_generic_updater.return_value.replace = MagicMock() + + print("Multi ASIC: {}".format(multi_asic.is_multi_asic())) + # Invocation of the command with the CliRunner + result = self.runner.invoke(config.config.commands["replace"], + [self.replace_file_path, + "--scope", "localhost"], + catch_exceptions=True) + + print("Exit Code: {}, output: {}".format(result.exit_code, result.output)) + # Assertions and verifications + self.assertEqual(result.exit_code, 0, "Command should succeed") + self.assertIn("Config replaced successfully.", result.output) + + # Verify mocked_open was called as expected + mocked_open.assert_called_with(self.replace_file_path, 'r') + + def test_repalce_multiasic_missing_scope(self): + # Mock open to simulate file reading + mock_replace_content = "[]" + with patch('builtins.open', mock_open(read_data=json.dumps(mock_replace_content)), create=True) as mocked_open: + # Mock GenericUpdater to avoid actual patch application + with patch('config.main.GenericUpdater') as mock_generic_updater: + mock_generic_updater.return_value.replace = MagicMock() + + print("Multi ASIC: {}".format(multi_asic.is_multi_asic())) + # Invocation of the command with the CliRunner + result = self.runner.invoke(config.config.commands["replace"], + [self.replace_file_path], + catch_exceptions=True) + + print("Exit Code: {}, output: {}".format(result.exit_code, result.output)) + # Assertions and verifications + self.assertEqual(result.exit_code, 2, "Command should failed") + self.assertIn("Missing option \"-s\"", result.output) + + def test_repalce_multiasic_with_wrong_scope(self): + # Mock open to simulate file reading + with patch('builtins.open', mock_open(read_data=json.dumps(self.patch_content)), create=True) as mocked_open: + # Mock GenericUpdater to avoid actual patch application + with patch('config.main.GenericUpdater') as mock_generic_updater: + mock_generic_updater.return_value.replace = MagicMock() + + print("Multi ASIC: {}".format(multi_asic.is_multi_asic())) + # Invocation of the command with the CliRunner + result = self.runner.invoke(config.config.commands["replace"], + [self.replace_file_path, + "--scope", "x"], + catch_exceptions=True) + + print("Exit Code: {}, output: {}".format(result.exit_code, result.output)) + # Assertions and verifications + self.assertEqual(result.exit_code, 2, "Command should failed") + self.assertIn("Invalid value for \"-s\"", result.output) + + def test_checkpoint_multiasic(self): + # Mock open to simulate file reading + checkpointname = "checkpointname" + # Mock GenericUpdater to avoid actual patch application + with patch('config.main.GenericUpdater') as mock_generic_updater: + mock_generic_updater.return_value.checkpoint = MagicMock() + + print("Multi ASIC: {}".format(multi_asic.is_multi_asic())) + # Invocation of the command with the CliRunner + result = self.runner.invoke(config.config.commands["checkpoint"], + [checkpointname], + catch_exceptions=True) + + print("Exit Code: {}, output: {}".format(result.exit_code, result.output)) + # Assertions and verifications + self.assertEqual(result.exit_code, 0, "Command should succeed") + self.assertIn("Checkpoint created successfully.", result.output) + + def test_rollback_multiasic(self): + # Mock open to simulate file reading + checkpointname = "checkpointname" + # Mock GenericUpdater to avoid actual patch application + with patch('config.main.GenericUpdater') as mock_generic_updater: + mock_generic_updater.return_value.rollback = MagicMock() + + print("Multi ASIC: {}".format(multi_asic.is_multi_asic())) + # Invocation of the command with the CliRunner + result = self.runner.invoke(config.config.commands["rollback"], + [checkpointname], + catch_exceptions=True) + + print("Exit Code: {}, output: {}".format(result.exit_code, result.output)) + # Assertions and verifications + self.assertEqual(result.exit_code, 0, "Command should succeed") + self.assertIn("Config rolled back successfully.", result.output) + + def test_delete_checkpoint_multiasic(self): + # Mock open to simulate file reading + checkpointname = "checkpointname" + # Mock GenericUpdater to avoid actual patch application + with patch('config.main.GenericUpdater') as mock_generic_updater: + mock_generic_updater.return_value.deletecheckpoint = MagicMock() + + print("Multi ASIC: {}".format(multi_asic.is_multi_asic())) + # Invocation of the command with the CliRunner + result = self.runner.invoke(config.config.commands["delete-checkpoint"], + [checkpointname], + catch_exceptions=True) + + print("Exit Code: {}, output: {}".format(result.exit_code, result.output)) + # Assertions and verifications + self.assertEqual(result.exit_code, 0, "Command should succeed") + self.assertIn("Checkpoint deleted successfully.", result.output) @classmethod def teardown_class(cls): @@ -2741,4 +2857,4 @@ def teardown_class(cls): from .mock_tables import dbconnector from .mock_tables import mock_single_asic importlib.reload(mock_single_asic) - dbconnector.load_database_config() \ No newline at end of file + dbconnector.load_database_config() From a3220046e5005000c908980c7c4f464cbf97a7c1 Mon Sep 17 00:00:00 2001 From: Xincun Li Date: Tue, 30 Apr 2024 17:02:34 -0700 Subject: [PATCH 02/44] Fix scope --- generic_config_updater/generic_updater.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/generic_config_updater/generic_updater.py b/generic_config_updater/generic_updater.py index 20fad576ca..5d36112a25 100644 --- a/generic_config_updater/generic_updater.py +++ b/generic_config_updater/generic_updater.py @@ -294,18 +294,15 @@ def replace(self, target_config): self.decorated_config_replacer.replace(target_config) def rollback(self, checkpoint_name): - checkpoint_name = checkpoint_name + self.namespace self.decorated_config_rollbacker.rollback(checkpoint_name) def checkpoint(self, checkpoint_name): - checkpoint_name = checkpoint_name + self.namespace self.decorated_config_rollbacker.checkpoint(checkpoint_name) def list_checkpoints(self): return self.decorated_config_rollbacker.list_checkpoints() def delete_checkpoint(self, checkpoint_name): - checkpoint_name = checkpoint_name + self.namespace self.decorated_config_rollbacker.delete_checkpoint(checkpoint_name) @@ -344,11 +341,9 @@ def replace(self, target_config): self.execute_write_action(Decorator.replace, self, target_config) def rollback(self, checkpoint_name): - checkpoint_name = checkpoint_name + self.namespace self.execute_write_action(Decorator.rollback, self, checkpoint_name) def checkpoint(self, checkpoint_name): - checkpoint_name = checkpoint_name + self.namespace self.execute_write_action(Decorator.checkpoint, self, checkpoint_name) def execute_write_action(self, action, *args): @@ -483,17 +478,14 @@ def replace(self, target_config, config_format, verbose, dry_run, ignore_non_yan config_replacer.replace(target_config) def rollback(self, checkpoint_name, verbose, dry_run, ignore_non_yang_tables, ignore_paths): - checkpoint_name = checkpoint_name + self.namespace config_rollbacker = self.generic_update_factory.create_config_rollbacker(verbose, dry_run, ignore_non_yang_tables, ignore_paths) config_rollbacker.rollback(checkpoint_name) def checkpoint(self, checkpoint_name, verbose): - checkpoint_name = checkpoint_name + self.namespace config_rollbacker = self.generic_update_factory.create_config_rollbacker(verbose) config_rollbacker.checkpoint(checkpoint_name) def delete_checkpoint(self, checkpoint_name, verbose): - checkpoint_name = checkpoint_name + self.namespace config_rollbacker = self.generic_update_factory.create_config_rollbacker(verbose) config_rollbacker.delete_checkpoint(checkpoint_name) From 406f3f9df9e7e98600c0f1952977ef25cb3ea5e7 Mon Sep 17 00:00:00 2001 From: Xincun Li Date: Tue, 30 Apr 2024 18:31:44 -0700 Subject: [PATCH 03/44] Fix syntax --- config/main.py | 8 +++--- tests/config_test.py | 64 +++++++++++++++++++------------------------- 2 files changed, 32 insertions(+), 40 deletions(-) diff --git a/config/main.py b/config/main.py index 61ffc1a03c..7cd186e3b5 100644 --- a/config/main.py +++ b/config/main.py @@ -1421,9 +1421,10 @@ def apply_patch(ctx, patch_file_path, format, dry_run, ignore_non_yang_tables, i @config.command() @click.argument('target-file-path', type=str, required=True) -@click.option('-s', '--scope', - help='Specify the namespace for Multi-ASIC environments. For single ASIC environments or Multi-ASIC localhost, specifying the namespace is not required.', - required=True if multi_asic.is_multi_asic() else False, +@click.option('-s', '--scope', + help='Specify the namespace for Multi-ASIC environments. For single ASIC environments or Multi-ASIC localhost, \ + specifying the namespace is not required.', + required=True if multi_asic.is_multi_asic() else False, type=click.Choice(multi_asic.get_namespace_list() + ['localhost'])) @click.option('-f', '--format', type=click.Choice([e.name for e in ConfigFormat]), default=ConfigFormat.CONFIGDB.name, @@ -1442,7 +1443,6 @@ def replace(ctx, target_file_path, scope, format, dry_run, ignore_non_yang_table **WARNING** The target config file should be the whole config, not just the part intended to be updated. : Path to the target file on the file-system. - If the device is a Multi-ASIC environment, please give the namespace of specific ASIC, for instance: localhost, asic0, asic1, ... """ diff --git a/tests/config_test.py b/tests/config_test.py index 05a23a336f..fa37dd8c6e 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -2732,7 +2732,7 @@ def test_apply_patch_dryrun_multiasic(self): # Ensure ConfigDBConnector was never instantiated or called mock_config_db_connector.assert_not_called() - + def test_repalce_multiasic(self): # Mock open to simulate file reading mock_replace_content = "[]" @@ -2755,47 +2755,41 @@ def test_repalce_multiasic(self): # Verify mocked_open was called as expected mocked_open.assert_called_with(self.replace_file_path, 'r') - + def test_repalce_multiasic_missing_scope(self): - # Mock open to simulate file reading - mock_replace_content = "[]" - with patch('builtins.open', mock_open(read_data=json.dumps(mock_replace_content)), create=True) as mocked_open: - # Mock GenericUpdater to avoid actual patch application - with patch('config.main.GenericUpdater') as mock_generic_updater: - mock_generic_updater.return_value.replace = MagicMock() + # Mock GenericUpdater to avoid actual patch application + with patch('config.main.GenericUpdater') as mock_generic_updater: + mock_generic_updater.return_value.replace = MagicMock() - print("Multi ASIC: {}".format(multi_asic.is_multi_asic())) - # Invocation of the command with the CliRunner - result = self.runner.invoke(config.config.commands["replace"], - [self.replace_file_path], - catch_exceptions=True) + print("Multi ASIC: {}".format(multi_asic.is_multi_asic())) + # Invocation of the command with the CliRunner + result = self.runner.invoke(config.config.commands["replace"], + [self.replace_file_path], + catch_exceptions=True) - print("Exit Code: {}, output: {}".format(result.exit_code, result.output)) - # Assertions and verifications - self.assertEqual(result.exit_code, 2, "Command should failed") - self.assertIn("Missing option \"-s\"", result.output) + print("Exit Code: {}, output: {}".format(result.exit_code, result.output)) + # Assertions and verifications + self.assertEqual(result.exit_code, 2, "Command should failed") + self.assertIn("Missing option \"-s\"", result.output) def test_repalce_multiasic_with_wrong_scope(self): - # Mock open to simulate file reading - with patch('builtins.open', mock_open(read_data=json.dumps(self.patch_content)), create=True) as mocked_open: - # Mock GenericUpdater to avoid actual patch application - with patch('config.main.GenericUpdater') as mock_generic_updater: - mock_generic_updater.return_value.replace = MagicMock() + # Mock GenericUpdater to avoid actual patch application + with patch('config.main.GenericUpdater') as mock_generic_updater: + mock_generic_updater.return_value.replace = MagicMock() - print("Multi ASIC: {}".format(multi_asic.is_multi_asic())) - # Invocation of the command with the CliRunner - result = self.runner.invoke(config.config.commands["replace"], - [self.replace_file_path, - "--scope", "x"], - catch_exceptions=True) + print("Multi ASIC: {}".format(multi_asic.is_multi_asic())) + # Invocation of the command with the CliRunner + result = self.runner.invoke(config.config.commands["replace"], + [self.replace_file_path, + "--scope", "x"], + catch_exceptions=True) + + print("Exit Code: {}, output: {}".format(result.exit_code, result.output)) + # Assertions and verifications + self.assertEqual(result.exit_code, 2, "Command should failed") + self.assertIn("Invalid value for \"-s\"", result.output) - print("Exit Code: {}, output: {}".format(result.exit_code, result.output)) - # Assertions and verifications - self.assertEqual(result.exit_code, 2, "Command should failed") - self.assertIn("Invalid value for \"-s\"", result.output) - def test_checkpoint_multiasic(self): - # Mock open to simulate file reading checkpointname = "checkpointname" # Mock GenericUpdater to avoid actual patch application with patch('config.main.GenericUpdater') as mock_generic_updater: @@ -2813,7 +2807,6 @@ def test_checkpoint_multiasic(self): self.assertIn("Checkpoint created successfully.", result.output) def test_rollback_multiasic(self): - # Mock open to simulate file reading checkpointname = "checkpointname" # Mock GenericUpdater to avoid actual patch application with patch('config.main.GenericUpdater') as mock_generic_updater: @@ -2831,7 +2824,6 @@ def test_rollback_multiasic(self): self.assertIn("Config rolled back successfully.", result.output) def test_delete_checkpoint_multiasic(self): - # Mock open to simulate file reading checkpointname = "checkpointname" # Mock GenericUpdater to avoid actual patch application with patch('config.main.GenericUpdater') as mock_generic_updater: From 029d0d0f1439616851c4ab997cd1e6b320686017 Mon Sep 17 00:00:00 2001 From: Xincun Li Date: Tue, 30 Apr 2024 18:35:54 -0700 Subject: [PATCH 04/44] Fix pre commit check --- config/main.py | 8 +++++--- generic_config_updater/generic_updater.py | 10 +++++----- tests/config_test.py | 6 +++--- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/config/main.py b/config/main.py index 7cd186e3b5..6ade9fba00 100644 --- a/config/main.py +++ b/config/main.py @@ -1444,7 +1444,7 @@ def replace(ctx, target_file_path, scope, format, dry_run, ignore_non_yang_table : Path to the target file on the file-system. If the device is a Multi-ASIC environment, please give the namespace of specific ASIC, for instance: - localhost, asic0, asic1, ... + localhost, asic0, asic1, ... """ try: print_dry_run_message(dry_run) @@ -1460,7 +1460,8 @@ def replace(ctx, target_file_path, scope, format, dry_run, ignore_non_yang_table scope = scope if scope != "localhost" else multi_asic.DEFAULT_NAMESPACE else: scope = multi_asic.DEFAULT_NAMESPACE - GenericUpdater(namespace=scope).replace(target_config, config_format, verbose, dry_run, ignore_non_yang_tables, ignore_path) + GenericUpdater(namespace=scope).replace(target_config, config_format, verbose, dry_run, + ignore_non_yang_tables, ignore_path) click.secho("Config replaced successfully.", fg="cyan", underline=True) except Exception as ex: @@ -1486,7 +1487,8 @@ def rollback(ctx, checkpoint_name, dry_run, ignore_non_yang_tables, ignore_path, if multi_asic.is_multi_asic(): asic_list.extend(multi_asic.get_namespace_list()) for asic in asic_list: - GenericUpdater(namespace=asic).rollback(checkpoint_name, verbose, dry_run, ignore_non_yang_tables, ignore_path) + GenericUpdater(namespace=asic).rollback(checkpoint_name, verbose, dry_run, + ignore_non_yang_tables, ignore_path) click.secho("Config rolled back successfully.", fg="cyan", underline=True) except Exception as ex: diff --git a/generic_config_updater/generic_updater.py b/generic_config_updater/generic_updater.py index 5d36112a25..5807aa007b 100644 --- a/generic_config_updater/generic_updater.py +++ b/generic_config_updater/generic_updater.py @@ -84,14 +84,14 @@ def apply(self, patch, sort=True): self.config_wrapper.validate_field_operation(old_config, target_config) # Validate target config does not have empty tables since they do not show up in ConfigDb - self.logger.log_notice(f"{scope}: validating target config does not have empty tables, " \ - "since they do not show up in ConfigDb.") + self.logger.log_notice(f"{scope}: validating target config does not have empty tables, + since they do not show up in ConfigDb.") empty_tables = self.config_wrapper.get_empty_tables(target_config) if empty_tables: # if there are empty tables empty_tables_txt = ", ".join(empty_tables) - raise EmptyTableError(f"{scope}: given patch is not valid because it will result in empty tables " \ - "which is not allowed in ConfigDb. " \ - f"Table{'s' if len(empty_tables) != 1 else ''}: {empty_tables_txt}") + raise EmptyTableError(f"{scope}: given patch is not valid because it will result in empty tables + which is not allowed in ConfigDb. + Table{'s' if len(empty_tables) != 1 else ''}: {empty_tables_txt}") # Generate list of changes to apply if sort: diff --git a/tests/config_test.py b/tests/config_test.py index fa37dd8c6e..72f6693a96 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -2771,7 +2771,7 @@ def test_repalce_multiasic_missing_scope(self): # Assertions and verifications self.assertEqual(result.exit_code, 2, "Command should failed") self.assertIn("Missing option \"-s\"", result.output) - + def test_repalce_multiasic_with_wrong_scope(self): # Mock GenericUpdater to avoid actual patch application with patch('config.main.GenericUpdater') as mock_generic_updater: @@ -2805,7 +2805,7 @@ def test_checkpoint_multiasic(self): # Assertions and verifications self.assertEqual(result.exit_code, 0, "Command should succeed") self.assertIn("Checkpoint created successfully.", result.output) - + def test_rollback_multiasic(self): checkpointname = "checkpointname" # Mock GenericUpdater to avoid actual patch application @@ -2822,7 +2822,7 @@ def test_rollback_multiasic(self): # Assertions and verifications self.assertEqual(result.exit_code, 0, "Command should succeed") self.assertIn("Config rolled back successfully.", result.output) - + def test_delete_checkpoint_multiasic(self): checkpointname = "checkpointname" # Mock GenericUpdater to avoid actual patch application From 3aec0f93d64a6265b095b8208841231e7048fa3e Mon Sep 17 00:00:00 2001 From: Xincun Li Date: Tue, 30 Apr 2024 18:39:45 -0700 Subject: [PATCH 05/44] fix pre commit test --- generic_config_updater/generic_updater.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/generic_config_updater/generic_updater.py b/generic_config_updater/generic_updater.py index 5807aa007b..8e5c7204d8 100644 --- a/generic_config_updater/generic_updater.py +++ b/generic_config_updater/generic_updater.py @@ -84,13 +84,13 @@ def apply(self, patch, sort=True): self.config_wrapper.validate_field_operation(old_config, target_config) # Validate target config does not have empty tables since they do not show up in ConfigDb - self.logger.log_notice(f"{scope}: validating target config does not have empty tables, + self.logger.log_notice(f"{scope}: validating target config does not have empty tables, \ since they do not show up in ConfigDb.") empty_tables = self.config_wrapper.get_empty_tables(target_config) if empty_tables: # if there are empty tables empty_tables_txt = ", ".join(empty_tables) - raise EmptyTableError(f"{scope}: given patch is not valid because it will result in empty tables - which is not allowed in ConfigDb. + raise EmptyTableError(f"{scope}: given patch is not valid because it will result in empty tables \ + which is not allowed in ConfigDb. \ Table{'s' if len(empty_tables) != 1 else ''}: {empty_tables_txt}") # Generate list of changes to apply From b2962000ed01fa72d946d8cf73b1d507b7b54229 Mon Sep 17 00:00:00 2001 From: Xincun Li <147451452+xincunli-sonic@users.noreply.github.com> Date: Wed, 1 May 2024 09:09:39 -0700 Subject: [PATCH 06/44] Refactor scope parameter comment --- config/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/main.py b/config/main.py index 6ade9fba00..cab4ee5962 100644 --- a/config/main.py +++ b/config/main.py @@ -1422,7 +1422,7 @@ def apply_patch(ctx, patch_file_path, format, dry_run, ignore_non_yang_tables, i @config.command() @click.argument('target-file-path', type=str, required=True) @click.option('-s', '--scope', - help='Specify the namespace for Multi-ASIC environments. For single ASIC environments or Multi-ASIC localhost, \ + help='Specify the namespace for Multi-ASIC environments. For Single-ASIC environments, \ specifying the namespace is not required.', required=True if multi_asic.is_multi_asic() else False, type=click.Choice(multi_asic.get_namespace_list() + ['localhost'])) From 3eabf79713db92d565da10b254284546b721a410 Mon Sep 17 00:00:00 2001 From: Xincun Li Date: Tue, 30 Apr 2024 15:41:37 -0700 Subject: [PATCH 07/44] Add Checkpoint and Rollback for Multi ASIC. --- config/main.py | 42 ++++++-- generic_config_updater/generic_updater.py | 11 ++ tests/config_test.py | 118 +++++++++++++++++++++- 3 files changed, 161 insertions(+), 10 deletions(-) diff --git a/config/main.py b/config/main.py index b64e93949c..527df7ca03 100644 --- a/config/main.py +++ b/config/main.py @@ -1178,7 +1178,7 @@ def apply_patch_for_scope(scope_changes, results, config_format, verbose, dry_ru log.log_notice(f"'apply-patch' executed successfully for {scope_for_log} by {changes}") except Exception as e: results[scope_for_log] = {"success": False, "message": str(e)} - log.log_error(f"'apply-patch' executed failed for {scope_for_log} by {changes} due to {str(e)}") + log.log_warning(f"'apply-patch' executed failed for {scope_for_log} by {changes} due to {str(e)}") def validate_patch(patch): @@ -1466,6 +1466,10 @@ def apply_patch(ctx, patch_file_path, format, dry_run, ignore_non_yang_tables, i @config.command() @click.argument('target-file-path', type=str, required=True) +@click.option('-s', '--scope', + help='Specify the namespace for Multi-ASIC environments. For single ASIC environments or Multi-ASIC localhost, specifying the namespace is not required.', + required=True if multi_asic.is_multi_asic() else False, + type=click.Choice(multi_asic.get_namespace_list() + ['localhost'])) @click.option('-f', '--format', type=click.Choice([e.name for e in ConfigFormat]), default=ConfigFormat.CONFIGDB.name, help='format of target config is either ConfigDb(ABNF) or SonicYang', @@ -1475,14 +1479,18 @@ def apply_patch(ctx, patch_file_path, format, dry_run, ignore_non_yang_tables, i @click.option('-i', '--ignore-path', multiple=True, help='ignore validation for config specified by given path which is a JsonPointer', hidden=True) @click.option('-v', '--verbose', is_flag=True, default=False, help='print additional details of what the operation is doing') @click.pass_context -def replace(ctx, target_file_path, format, dry_run, ignore_non_yang_tables, ignore_path, verbose): +def replace(ctx, target_file_path, scope, format, dry_run, ignore_non_yang_tables, ignore_path, verbose): """Replace the whole config with the specified config. The config is replaced with minimum disruption e.g. if ACL config is different between current and target config only ACL config is updated, and other config/services such as DHCP will not be affected. **WARNING** The target config file should be the whole config, not just the part intended to be updated. - : Path to the target file on the file-system.""" + : Path to the target file on the file-system. + + If the device is a Multi-ASIC environment, please give the namespace of specific ASIC, for instance: + localhost, asic0, asic1, ... + """ try: print_dry_run_message(dry_run) @@ -1491,8 +1499,13 @@ def replace(ctx, target_file_path, format, dry_run, ignore_non_yang_tables, igno target_config = json.loads(target_config_as_text) config_format = ConfigFormat[format.upper()] - - GenericUpdater().replace(target_config, config_format, verbose, dry_run, ignore_non_yang_tables, ignore_path) + if multi_asic.is_multi_asic(): + if scope not in (multi_asic.get_namespace_list() + ["localhost"]): + raise Exception(f"Failed to replace config due to wrong namespace:{scope}") + scope = scope if scope != "localhost" else multi_asic.DEFAULT_NAMESPACE + else: + scope = multi_asic.DEFAULT_NAMESPACE + GenericUpdater(namespace=scope).replace(target_config, config_format, verbose, dry_run, ignore_non_yang_tables, ignore_path) click.secho("Config replaced successfully.", fg="cyan", underline=True) except Exception as ex: @@ -1514,8 +1527,11 @@ def rollback(ctx, checkpoint_name, dry_run, ignore_non_yang_tables, ignore_path, : The checkpoint name, use `config list-checkpoints` command to see available checkpoints.""" try: print_dry_run_message(dry_run) - - GenericUpdater().rollback(checkpoint_name, verbose, dry_run, ignore_non_yang_tables, ignore_path) + asic_list = [multi_asic.DEFAULT_NAMESPACE] + if multi_asic.is_multi_asic(): + asic_list.extend(multi_asic.get_namespace_list()) + for asic in asic_list: + GenericUpdater(namespace=asic).rollback(checkpoint_name, verbose, dry_run, ignore_non_yang_tables, ignore_path) click.secho("Config rolled back successfully.", fg="cyan", underline=True) except Exception as ex: @@ -1531,7 +1547,11 @@ def checkpoint(ctx, checkpoint_name, verbose): : The checkpoint name, use `config list-checkpoints` command to see available checkpoints.""" try: - GenericUpdater().checkpoint(checkpoint_name, verbose) + asic_list = [multi_asic.DEFAULT_NAMESPACE] + if multi_asic.is_multi_asic(): + asic_list.extend(multi_asic.get_namespace_list()) + for asic in asic_list: + GenericUpdater(namespace=asic).checkpoint(checkpoint_name, verbose) click.secho("Checkpoint created successfully.", fg="cyan", underline=True) except Exception as ex: @@ -1547,7 +1567,11 @@ def delete_checkpoint(ctx, checkpoint_name, verbose): : The checkpoint name, use `config list-checkpoints` command to see available checkpoints.""" try: - GenericUpdater().delete_checkpoint(checkpoint_name, verbose) + asic_list = [multi_asic.DEFAULT_NAMESPACE] + if multi_asic.is_multi_asic(): + asic_list.extend(multi_asic.get_namespace_list()) + for asic in asic_list: + GenericUpdater(namespace=asic).delete_checkpoint(checkpoint_name, verbose) click.secho("Checkpoint deleted successfully.", fg="cyan", underline=True) except Exception as ex: diff --git a/generic_config_updater/generic_updater.py b/generic_config_updater/generic_updater.py index 374ce7670c..04f8976e7c 100644 --- a/generic_config_updater/generic_updater.py +++ b/generic_config_updater/generic_updater.py @@ -164,6 +164,7 @@ def __init__(self, self.config_wrapper = config_wrapper if config_wrapper is not None else ConfigWrapper(namespace=self.namespace) def rollback(self, checkpoint_name): + checkpoint_name = checkpoint_name + self.namespace self.logger.log_notice("Config rollbacking starting.") self.logger.log_notice(f"Checkpoint name: {checkpoint_name}.") @@ -180,6 +181,7 @@ def rollback(self, checkpoint_name): self.logger.log_notice("Config rollbacking completed.") def checkpoint(self, checkpoint_name): + checkpoint_name = checkpoint_name + self.namespace self.logger.log_notice("Config checkpoint starting.") self.logger.log_notice(f"Checkpoint name: {checkpoint_name}.") @@ -218,6 +220,7 @@ def list_checkpoints(self): return checkpoint_names def delete_checkpoint(self, checkpoint_name): + checkpoint_name = checkpoint_name + self.namespace self.logger.log_notice("Deleting checkpoint starting.") self.logger.log_notice(f"Checkpoint name: {checkpoint_name}.") @@ -286,15 +289,18 @@ def replace(self, target_config): self.decorated_config_replacer.replace(target_config) def rollback(self, checkpoint_name): + checkpoint_name = checkpoint_name + self.namespace self.decorated_config_rollbacker.rollback(checkpoint_name) def checkpoint(self, checkpoint_name): + checkpoint_name = checkpoint_name + self.namespace self.decorated_config_rollbacker.checkpoint(checkpoint_name) def list_checkpoints(self): return self.decorated_config_rollbacker.list_checkpoints() def delete_checkpoint(self, checkpoint_name): + checkpoint_name = checkpoint_name + self.namespace self.decorated_config_rollbacker.delete_checkpoint(checkpoint_name) @@ -333,9 +339,11 @@ def replace(self, target_config): self.execute_write_action(Decorator.replace, self, target_config) def rollback(self, checkpoint_name): + checkpoint_name = checkpoint_name + self.namespace self.execute_write_action(Decorator.rollback, self, checkpoint_name) def checkpoint(self, checkpoint_name): + checkpoint_name = checkpoint_name + self.namespace self.execute_write_action(Decorator.checkpoint, self, checkpoint_name) def execute_write_action(self, action, *args): @@ -470,14 +478,17 @@ def replace(self, target_config, config_format, verbose, dry_run, ignore_non_yan config_replacer.replace(target_config) def rollback(self, checkpoint_name, verbose, dry_run, ignore_non_yang_tables, ignore_paths): + checkpoint_name = checkpoint_name + self.namespace config_rollbacker = self.generic_update_factory.create_config_rollbacker(verbose, dry_run, ignore_non_yang_tables, ignore_paths) config_rollbacker.rollback(checkpoint_name) def checkpoint(self, checkpoint_name, verbose): + checkpoint_name = checkpoint_name + self.namespace config_rollbacker = self.generic_update_factory.create_config_rollbacker(verbose) config_rollbacker.checkpoint(checkpoint_name) def delete_checkpoint(self, checkpoint_name, verbose): + checkpoint_name = checkpoint_name + self.namespace config_rollbacker = self.generic_update_factory.create_config_rollbacker(verbose) config_rollbacker.delete_checkpoint(checkpoint_name) diff --git a/tests/config_test.py b/tests/config_test.py index 1e5f7a9009..3eb636c3ad 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -2691,6 +2691,7 @@ def setUp(self): self.runner = CliRunner() self.patch_file_path = 'path/to/patch.json' + self.replace_file_path = 'path/to/replace.json' self.patch_content = [ { "op": "add", @@ -2782,6 +2783,121 @@ def test_apply_patch_dryrun_multiasic(self): # Ensure ConfigDBConnector was never instantiated or called mock_config_db_connector.assert_not_called() + + def test_repalce_multiasic(self): + # Mock open to simulate file reading + mock_replace_content = "[]" + with patch('builtins.open', mock_open(read_data=json.dumps(mock_replace_content)), create=True) as mocked_open: + # Mock GenericUpdater to avoid actual patch application + with patch('config.main.GenericUpdater') as mock_generic_updater: + mock_generic_updater.return_value.replace = MagicMock() + + print("Multi ASIC: {}".format(multi_asic.is_multi_asic())) + # Invocation of the command with the CliRunner + result = self.runner.invoke(config.config.commands["replace"], + [self.replace_file_path, + "--scope", "localhost"], + catch_exceptions=True) + + print("Exit Code: {}, output: {}".format(result.exit_code, result.output)) + # Assertions and verifications + self.assertEqual(result.exit_code, 0, "Command should succeed") + self.assertIn("Config replaced successfully.", result.output) + + # Verify mocked_open was called as expected + mocked_open.assert_called_with(self.replace_file_path, 'r') + + def test_repalce_multiasic_missing_scope(self): + # Mock open to simulate file reading + mock_replace_content = "[]" + with patch('builtins.open', mock_open(read_data=json.dumps(mock_replace_content)), create=True) as mocked_open: + # Mock GenericUpdater to avoid actual patch application + with patch('config.main.GenericUpdater') as mock_generic_updater: + mock_generic_updater.return_value.replace = MagicMock() + + print("Multi ASIC: {}".format(multi_asic.is_multi_asic())) + # Invocation of the command with the CliRunner + result = self.runner.invoke(config.config.commands["replace"], + [self.replace_file_path], + catch_exceptions=True) + + print("Exit Code: {}, output: {}".format(result.exit_code, result.output)) + # Assertions and verifications + self.assertEqual(result.exit_code, 2, "Command should failed") + self.assertIn("Missing option \"-s\"", result.output) + + def test_repalce_multiasic_with_wrong_scope(self): + # Mock open to simulate file reading + with patch('builtins.open', mock_open(read_data=json.dumps(self.patch_content)), create=True) as mocked_open: + # Mock GenericUpdater to avoid actual patch application + with patch('config.main.GenericUpdater') as mock_generic_updater: + mock_generic_updater.return_value.replace = MagicMock() + + print("Multi ASIC: {}".format(multi_asic.is_multi_asic())) + # Invocation of the command with the CliRunner + result = self.runner.invoke(config.config.commands["replace"], + [self.replace_file_path, + "--scope", "x"], + catch_exceptions=True) + + print("Exit Code: {}, output: {}".format(result.exit_code, result.output)) + # Assertions and verifications + self.assertEqual(result.exit_code, 2, "Command should failed") + self.assertIn("Invalid value for \"-s\"", result.output) + + def test_checkpoint_multiasic(self): + # Mock open to simulate file reading + checkpointname = "checkpointname" + # Mock GenericUpdater to avoid actual patch application + with patch('config.main.GenericUpdater') as mock_generic_updater: + mock_generic_updater.return_value.checkpoint = MagicMock() + + print("Multi ASIC: {}".format(multi_asic.is_multi_asic())) + # Invocation of the command with the CliRunner + result = self.runner.invoke(config.config.commands["checkpoint"], + [checkpointname], + catch_exceptions=True) + + print("Exit Code: {}, output: {}".format(result.exit_code, result.output)) + # Assertions and verifications + self.assertEqual(result.exit_code, 0, "Command should succeed") + self.assertIn("Checkpoint created successfully.", result.output) + + def test_rollback_multiasic(self): + # Mock open to simulate file reading + checkpointname = "checkpointname" + # Mock GenericUpdater to avoid actual patch application + with patch('config.main.GenericUpdater') as mock_generic_updater: + mock_generic_updater.return_value.rollback = MagicMock() + + print("Multi ASIC: {}".format(multi_asic.is_multi_asic())) + # Invocation of the command with the CliRunner + result = self.runner.invoke(config.config.commands["rollback"], + [checkpointname], + catch_exceptions=True) + + print("Exit Code: {}, output: {}".format(result.exit_code, result.output)) + # Assertions and verifications + self.assertEqual(result.exit_code, 0, "Command should succeed") + self.assertIn("Config rolled back successfully.", result.output) + + def test_delete_checkpoint_multiasic(self): + # Mock open to simulate file reading + checkpointname = "checkpointname" + # Mock GenericUpdater to avoid actual patch application + with patch('config.main.GenericUpdater') as mock_generic_updater: + mock_generic_updater.return_value.deletecheckpoint = MagicMock() + + print("Multi ASIC: {}".format(multi_asic.is_multi_asic())) + # Invocation of the command with the CliRunner + result = self.runner.invoke(config.config.commands["delete-checkpoint"], + [checkpointname], + catch_exceptions=True) + + print("Exit Code: {}, output: {}".format(result.exit_code, result.output)) + # Assertions and verifications + self.assertEqual(result.exit_code, 0, "Command should succeed") + self.assertIn("Checkpoint deleted successfully.", result.output) @patch('config.main.subprocess.Popen') @patch('config.main.SonicYangCfgDbGenerator.validate_config_db_json', mock.Mock(return_value=True)) @@ -2883,4 +2999,4 @@ def teardown_class(cls): from .mock_tables import dbconnector from .mock_tables import mock_single_asic importlib.reload(mock_single_asic) - dbconnector.load_database_config() \ No newline at end of file + dbconnector.load_database_config() From 9fe2bd49509828845838a52cb236263f935b608d Mon Sep 17 00:00:00 2001 From: Xincun Li Date: Tue, 30 Apr 2024 17:02:34 -0700 Subject: [PATCH 08/44] Fix scope --- generic_config_updater/generic_updater.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/generic_config_updater/generic_updater.py b/generic_config_updater/generic_updater.py index 04f8976e7c..494eb36286 100644 --- a/generic_config_updater/generic_updater.py +++ b/generic_config_updater/generic_updater.py @@ -289,18 +289,15 @@ def replace(self, target_config): self.decorated_config_replacer.replace(target_config) def rollback(self, checkpoint_name): - checkpoint_name = checkpoint_name + self.namespace self.decorated_config_rollbacker.rollback(checkpoint_name) def checkpoint(self, checkpoint_name): - checkpoint_name = checkpoint_name + self.namespace self.decorated_config_rollbacker.checkpoint(checkpoint_name) def list_checkpoints(self): return self.decorated_config_rollbacker.list_checkpoints() def delete_checkpoint(self, checkpoint_name): - checkpoint_name = checkpoint_name + self.namespace self.decorated_config_rollbacker.delete_checkpoint(checkpoint_name) @@ -339,11 +336,9 @@ def replace(self, target_config): self.execute_write_action(Decorator.replace, self, target_config) def rollback(self, checkpoint_name): - checkpoint_name = checkpoint_name + self.namespace self.execute_write_action(Decorator.rollback, self, checkpoint_name) def checkpoint(self, checkpoint_name): - checkpoint_name = checkpoint_name + self.namespace self.execute_write_action(Decorator.checkpoint, self, checkpoint_name) def execute_write_action(self, action, *args): @@ -478,17 +473,14 @@ def replace(self, target_config, config_format, verbose, dry_run, ignore_non_yan config_replacer.replace(target_config) def rollback(self, checkpoint_name, verbose, dry_run, ignore_non_yang_tables, ignore_paths): - checkpoint_name = checkpoint_name + self.namespace config_rollbacker = self.generic_update_factory.create_config_rollbacker(verbose, dry_run, ignore_non_yang_tables, ignore_paths) config_rollbacker.rollback(checkpoint_name) def checkpoint(self, checkpoint_name, verbose): - checkpoint_name = checkpoint_name + self.namespace config_rollbacker = self.generic_update_factory.create_config_rollbacker(verbose) config_rollbacker.checkpoint(checkpoint_name) def delete_checkpoint(self, checkpoint_name, verbose): - checkpoint_name = checkpoint_name + self.namespace config_rollbacker = self.generic_update_factory.create_config_rollbacker(verbose) config_rollbacker.delete_checkpoint(checkpoint_name) From 5c1e8fa50e50bafc98f54cdcdc2aa7ebb05884aa Mon Sep 17 00:00:00 2001 From: Xincun Li Date: Tue, 30 Apr 2024 18:31:44 -0700 Subject: [PATCH 09/44] Fix syntax --- config/main.py | 8 +++--- tests/config_test.py | 64 +++++++++++++++++++------------------------- 2 files changed, 32 insertions(+), 40 deletions(-) diff --git a/config/main.py b/config/main.py index 527df7ca03..2584ccdc18 100644 --- a/config/main.py +++ b/config/main.py @@ -1466,9 +1466,10 @@ def apply_patch(ctx, patch_file_path, format, dry_run, ignore_non_yang_tables, i @config.command() @click.argument('target-file-path', type=str, required=True) -@click.option('-s', '--scope', - help='Specify the namespace for Multi-ASIC environments. For single ASIC environments or Multi-ASIC localhost, specifying the namespace is not required.', - required=True if multi_asic.is_multi_asic() else False, +@click.option('-s', '--scope', + help='Specify the namespace for Multi-ASIC environments. For single ASIC environments or Multi-ASIC localhost, \ + specifying the namespace is not required.', + required=True if multi_asic.is_multi_asic() else False, type=click.Choice(multi_asic.get_namespace_list() + ['localhost'])) @click.option('-f', '--format', type=click.Choice([e.name for e in ConfigFormat]), default=ConfigFormat.CONFIGDB.name, @@ -1487,7 +1488,6 @@ def replace(ctx, target_file_path, scope, format, dry_run, ignore_non_yang_table **WARNING** The target config file should be the whole config, not just the part intended to be updated. : Path to the target file on the file-system. - If the device is a Multi-ASIC environment, please give the namespace of specific ASIC, for instance: localhost, asic0, asic1, ... """ diff --git a/tests/config_test.py b/tests/config_test.py index 3eb636c3ad..fcbebc30ec 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -2783,7 +2783,7 @@ def test_apply_patch_dryrun_multiasic(self): # Ensure ConfigDBConnector was never instantiated or called mock_config_db_connector.assert_not_called() - + def test_repalce_multiasic(self): # Mock open to simulate file reading mock_replace_content = "[]" @@ -2806,47 +2806,41 @@ def test_repalce_multiasic(self): # Verify mocked_open was called as expected mocked_open.assert_called_with(self.replace_file_path, 'r') - + def test_repalce_multiasic_missing_scope(self): - # Mock open to simulate file reading - mock_replace_content = "[]" - with patch('builtins.open', mock_open(read_data=json.dumps(mock_replace_content)), create=True) as mocked_open: - # Mock GenericUpdater to avoid actual patch application - with patch('config.main.GenericUpdater') as mock_generic_updater: - mock_generic_updater.return_value.replace = MagicMock() + # Mock GenericUpdater to avoid actual patch application + with patch('config.main.GenericUpdater') as mock_generic_updater: + mock_generic_updater.return_value.replace = MagicMock() - print("Multi ASIC: {}".format(multi_asic.is_multi_asic())) - # Invocation of the command with the CliRunner - result = self.runner.invoke(config.config.commands["replace"], - [self.replace_file_path], - catch_exceptions=True) + print("Multi ASIC: {}".format(multi_asic.is_multi_asic())) + # Invocation of the command with the CliRunner + result = self.runner.invoke(config.config.commands["replace"], + [self.replace_file_path], + catch_exceptions=True) - print("Exit Code: {}, output: {}".format(result.exit_code, result.output)) - # Assertions and verifications - self.assertEqual(result.exit_code, 2, "Command should failed") - self.assertIn("Missing option \"-s\"", result.output) + print("Exit Code: {}, output: {}".format(result.exit_code, result.output)) + # Assertions and verifications + self.assertEqual(result.exit_code, 2, "Command should failed") + self.assertIn("Missing option \"-s\"", result.output) def test_repalce_multiasic_with_wrong_scope(self): - # Mock open to simulate file reading - with patch('builtins.open', mock_open(read_data=json.dumps(self.patch_content)), create=True) as mocked_open: - # Mock GenericUpdater to avoid actual patch application - with patch('config.main.GenericUpdater') as mock_generic_updater: - mock_generic_updater.return_value.replace = MagicMock() + # Mock GenericUpdater to avoid actual patch application + with patch('config.main.GenericUpdater') as mock_generic_updater: + mock_generic_updater.return_value.replace = MagicMock() - print("Multi ASIC: {}".format(multi_asic.is_multi_asic())) - # Invocation of the command with the CliRunner - result = self.runner.invoke(config.config.commands["replace"], - [self.replace_file_path, - "--scope", "x"], - catch_exceptions=True) + print("Multi ASIC: {}".format(multi_asic.is_multi_asic())) + # Invocation of the command with the CliRunner + result = self.runner.invoke(config.config.commands["replace"], + [self.replace_file_path, + "--scope", "x"], + catch_exceptions=True) + + print("Exit Code: {}, output: {}".format(result.exit_code, result.output)) + # Assertions and verifications + self.assertEqual(result.exit_code, 2, "Command should failed") + self.assertIn("Invalid value for \"-s\"", result.output) - print("Exit Code: {}, output: {}".format(result.exit_code, result.output)) - # Assertions and verifications - self.assertEqual(result.exit_code, 2, "Command should failed") - self.assertIn("Invalid value for \"-s\"", result.output) - def test_checkpoint_multiasic(self): - # Mock open to simulate file reading checkpointname = "checkpointname" # Mock GenericUpdater to avoid actual patch application with patch('config.main.GenericUpdater') as mock_generic_updater: @@ -2864,7 +2858,6 @@ def test_checkpoint_multiasic(self): self.assertIn("Checkpoint created successfully.", result.output) def test_rollback_multiasic(self): - # Mock open to simulate file reading checkpointname = "checkpointname" # Mock GenericUpdater to avoid actual patch application with patch('config.main.GenericUpdater') as mock_generic_updater: @@ -2882,7 +2875,6 @@ def test_rollback_multiasic(self): self.assertIn("Config rolled back successfully.", result.output) def test_delete_checkpoint_multiasic(self): - # Mock open to simulate file reading checkpointname = "checkpointname" # Mock GenericUpdater to avoid actual patch application with patch('config.main.GenericUpdater') as mock_generic_updater: From 9cdff4853eddd88a3cba70f91ba2c6ac4e205d11 Mon Sep 17 00:00:00 2001 From: Xincun Li Date: Tue, 30 Apr 2024 18:35:54 -0700 Subject: [PATCH 10/44] Fix pre commit check --- config/main.py | 8 +++++--- generic_config_updater/generic_updater.py | 6 +++--- tests/config_test.py | 6 +++--- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/config/main.py b/config/main.py index 2584ccdc18..556542af38 100644 --- a/config/main.py +++ b/config/main.py @@ -1489,7 +1489,7 @@ def replace(ctx, target_file_path, scope, format, dry_run, ignore_non_yang_table : Path to the target file on the file-system. If the device is a Multi-ASIC environment, please give the namespace of specific ASIC, for instance: - localhost, asic0, asic1, ... + localhost, asic0, asic1, ... """ try: print_dry_run_message(dry_run) @@ -1505,7 +1505,8 @@ def replace(ctx, target_file_path, scope, format, dry_run, ignore_non_yang_table scope = scope if scope != "localhost" else multi_asic.DEFAULT_NAMESPACE else: scope = multi_asic.DEFAULT_NAMESPACE - GenericUpdater(namespace=scope).replace(target_config, config_format, verbose, dry_run, ignore_non_yang_tables, ignore_path) + GenericUpdater(namespace=scope).replace(target_config, config_format, verbose, dry_run, + ignore_non_yang_tables, ignore_path) click.secho("Config replaced successfully.", fg="cyan", underline=True) except Exception as ex: @@ -1531,7 +1532,8 @@ def rollback(ctx, checkpoint_name, dry_run, ignore_non_yang_tables, ignore_path, if multi_asic.is_multi_asic(): asic_list.extend(multi_asic.get_namespace_list()) for asic in asic_list: - GenericUpdater(namespace=asic).rollback(checkpoint_name, verbose, dry_run, ignore_non_yang_tables, ignore_path) + GenericUpdater(namespace=asic).rollback(checkpoint_name, verbose, dry_run, + ignore_non_yang_tables, ignore_path) click.secho("Config rolled back successfully.", fg="cyan", underline=True) except Exception as ex: diff --git a/generic_config_updater/generic_updater.py b/generic_config_updater/generic_updater.py index 494eb36286..b01cdc586f 100644 --- a/generic_config_updater/generic_updater.py +++ b/generic_config_updater/generic_updater.py @@ -87,9 +87,9 @@ def apply(self, patch, sort=True): empty_tables = self.config_wrapper.get_empty_tables(target_config) if empty_tables: # if there are empty tables empty_tables_txt = ", ".join(empty_tables) - raise EmptyTableError(f"{scope}: given patch is not valid because it will result in empty tables " \ - "which is not allowed in ConfigDb. " \ - f"Table{'s' if len(empty_tables) != 1 else ''}: {empty_tables_txt}") + raise EmptyTableError(f"{scope}: given patch is not valid because it will result in empty tables + which is not allowed in ConfigDb. + Table{'s' if len(empty_tables) != 1 else ''}: {empty_tables_txt}") # Generate list of changes to apply if sort: diff --git a/tests/config_test.py b/tests/config_test.py index fcbebc30ec..e85ec1a10f 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -2822,7 +2822,7 @@ def test_repalce_multiasic_missing_scope(self): # Assertions and verifications self.assertEqual(result.exit_code, 2, "Command should failed") self.assertIn("Missing option \"-s\"", result.output) - + def test_repalce_multiasic_with_wrong_scope(self): # Mock GenericUpdater to avoid actual patch application with patch('config.main.GenericUpdater') as mock_generic_updater: @@ -2856,7 +2856,7 @@ def test_checkpoint_multiasic(self): # Assertions and verifications self.assertEqual(result.exit_code, 0, "Command should succeed") self.assertIn("Checkpoint created successfully.", result.output) - + def test_rollback_multiasic(self): checkpointname = "checkpointname" # Mock GenericUpdater to avoid actual patch application @@ -2873,7 +2873,7 @@ def test_rollback_multiasic(self): # Assertions and verifications self.assertEqual(result.exit_code, 0, "Command should succeed") self.assertIn("Config rolled back successfully.", result.output) - + def test_delete_checkpoint_multiasic(self): checkpointname = "checkpointname" # Mock GenericUpdater to avoid actual patch application From 6cdd6a287d1c1a5b758a5b9367b982199cceb641 Mon Sep 17 00:00:00 2001 From: Xincun Li Date: Tue, 30 Apr 2024 18:39:45 -0700 Subject: [PATCH 11/44] fix pre commit test --- generic_config_updater/generic_updater.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/generic_config_updater/generic_updater.py b/generic_config_updater/generic_updater.py index b01cdc586f..b68b7e0242 100644 --- a/generic_config_updater/generic_updater.py +++ b/generic_config_updater/generic_updater.py @@ -87,8 +87,8 @@ def apply(self, patch, sort=True): empty_tables = self.config_wrapper.get_empty_tables(target_config) if empty_tables: # if there are empty tables empty_tables_txt = ", ".join(empty_tables) - raise EmptyTableError(f"{scope}: given patch is not valid because it will result in empty tables - which is not allowed in ConfigDb. + raise EmptyTableError(f"{scope}: given patch is not valid because it will result in empty tables \ + which is not allowed in ConfigDb. \ Table{'s' if len(empty_tables) != 1 else ''}: {empty_tables_txt}") # Generate list of changes to apply From edfa2dd5abdf48ea8e705b18db087aa2ecbd1b8e Mon Sep 17 00:00:00 2001 From: Xincun Li <147451452+xincunli-sonic@users.noreply.github.com> Date: Wed, 1 May 2024 09:09:39 -0700 Subject: [PATCH 12/44] Refactor scope parameter comment --- config/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/main.py b/config/main.py index 556542af38..5b239971d8 100644 --- a/config/main.py +++ b/config/main.py @@ -1467,7 +1467,7 @@ def apply_patch(ctx, patch_file_path, format, dry_run, ignore_non_yang_tables, i @config.command() @click.argument('target-file-path', type=str, required=True) @click.option('-s', '--scope', - help='Specify the namespace for Multi-ASIC environments. For single ASIC environments or Multi-ASIC localhost, \ + help='Specify the namespace for Multi-ASIC environments. For Single-ASIC environments, \ specifying the namespace is not required.', required=True if multi_asic.is_multi_asic() else False, type=click.Choice(multi_asic.get_namespace_list() + ['localhost'])) From ab8fc1461b661af986293a7f0544424d97e6c61f Mon Sep 17 00:00:00 2001 From: Xincun Li Date: Fri, 17 May 2024 14:43:49 -0700 Subject: [PATCH 13/44] Refactor replace with single config file. --- config/main.py | 33 +++-- generic_config_updater/generic_updater.py | 2 +- tests/config_test.py | 144 ++++------------------ 3 files changed, 49 insertions(+), 130 deletions(-) diff --git a/config/main.py b/config/main.py index 5b239971d8..e8e34981ef 100644 --- a/config/main.py +++ b/config/main.py @@ -1178,7 +1178,7 @@ def apply_patch_for_scope(scope_changes, results, config_format, verbose, dry_ru log.log_notice(f"'apply-patch' executed successfully for {scope_for_log} by {changes}") except Exception as e: results[scope_for_log] = {"success": False, "message": str(e)} - log.log_warning(f"'apply-patch' executed failed for {scope_for_log} by {changes} due to {str(e)}") + log.log_error(f"'apply-patch' executed failed for {scope_for_log} by {changes} due to {str(e)}") def validate_patch(patch): @@ -1466,11 +1466,6 @@ def apply_patch(ctx, patch_file_path, format, dry_run, ignore_non_yang_tables, i @config.command() @click.argument('target-file-path', type=str, required=True) -@click.option('-s', '--scope', - help='Specify the namespace for Multi-ASIC environments. For Single-ASIC environments, \ - specifying the namespace is not required.', - required=True if multi_asic.is_multi_asic() else False, - type=click.Choice(multi_asic.get_namespace_list() + ['localhost'])) @click.option('-f', '--format', type=click.Choice([e.name for e in ConfigFormat]), default=ConfigFormat.CONFIGDB.name, help='format of target config is either ConfigDb(ABNF) or SonicYang', @@ -1480,7 +1475,7 @@ def apply_patch(ctx, patch_file_path, format, dry_run, ignore_non_yang_tables, i @click.option('-i', '--ignore-path', multiple=True, help='ignore validation for config specified by given path which is a JsonPointer', hidden=True) @click.option('-v', '--verbose', is_flag=True, default=False, help='print additional details of what the operation is doing') @click.pass_context -def replace(ctx, target_file_path, scope, format, dry_run, ignore_non_yang_tables, ignore_path, verbose): +def replace(ctx, target_file_path, format, dry_run, ignore_non_yang_tables, ignore_path, verbose): """Replace the whole config with the specified config. The config is replaced with minimum disruption e.g. if ACL config is different between current and target config only ACL config is updated, and other config/services such as DHCP will not be affected. @@ -1500,13 +1495,27 @@ def replace(ctx, target_file_path, scope, format, dry_run, ignore_non_yang_table config_format = ConfigFormat[format.upper()] if multi_asic.is_multi_asic(): - if scope not in (multi_asic.get_namespace_list() + ["localhost"]): - raise Exception(f"Failed to replace config due to wrong namespace:{scope}") - scope = scope if scope != "localhost" else multi_asic.DEFAULT_NAMESPACE + config_keys = set(target_config.keys()) + scope_set = set(multi_asic.get_namespace_list() + [HOST_NAMESPACE]) + missing_scopes = scope_set - config_keys + if missing_scopes: + raise GenericConfigUpdaterError(f"To be replace config: {target_file_path} is missing these namespaces: {missing_scopes}") + + for scope in scope_set: + scope_config = target_config.pop("scope") + if not SonicYangCfgDbGenerator().validate_config_db_json(scope_config): + raise GenericConfigUpdaterError(f"Invalid config for {scope} in {target_file_path}") + + if scope.lower() == HOST_NAMESPACE: + scope = multi_asic.DEFAULT_NAMESPACE + GenericUpdater(namespace=scope).replace(scope_config, config_format, verbose, dry_run, + ignore_non_yang_tables, ignore_path) else: + if not SonicYangCfgDbGenerator().validate_config_db_json(scope_config): + raise GenericConfigUpdaterError(f"Invalid config in {target_file_path}") scope = multi_asic.DEFAULT_NAMESPACE - GenericUpdater(namespace=scope).replace(target_config, config_format, verbose, dry_run, - ignore_non_yang_tables, ignore_path) + GenericUpdater(namespace=scope).replace(target_config, config_format, verbose, dry_run, + ignore_non_yang_tables, ignore_path) click.secho("Config replaced successfully.", fg="cyan", underline=True) except Exception as ex: diff --git a/generic_config_updater/generic_updater.py b/generic_config_updater/generic_updater.py index b724732f17..367eb97e53 100644 --- a/generic_config_updater/generic_updater.py +++ b/generic_config_updater/generic_updater.py @@ -84,7 +84,7 @@ def apply(self, patch, sort=True): # Validate target config does not have empty tables since they do not show up in ConfigDb self.logger.log_notice(f"""{scope}: validating target config does not have empty tables, since they do not show up in ConfigDb.""") - + empty_tables = self.config_wrapper.get_empty_tables(target_config) if empty_tables: # if there are empty tables empty_tables_txt = ", ".join(empty_tables) diff --git a/tests/config_test.py b/tests/config_test.py index 2f71f7e627..d4ba0fb9b8 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -2784,113 +2784,6 @@ def test_apply_patch_dryrun_multiasic(self): # Ensure ConfigDBConnector was never instantiated or called mock_config_db_connector.assert_not_called() - def test_repalce_multiasic(self): - # Mock open to simulate file reading - mock_replace_content = "[]" - with patch('builtins.open', mock_open(read_data=json.dumps(mock_replace_content)), create=True) as mocked_open: - # Mock GenericUpdater to avoid actual patch application - with patch('config.main.GenericUpdater') as mock_generic_updater: - mock_generic_updater.return_value.replace = MagicMock() - - print("Multi ASIC: {}".format(multi_asic.is_multi_asic())) - # Invocation of the command with the CliRunner - result = self.runner.invoke(config.config.commands["replace"], - [self.replace_file_path, - "--scope", "localhost"], - catch_exceptions=True) - - print("Exit Code: {}, output: {}".format(result.exit_code, result.output)) - # Assertions and verifications - self.assertEqual(result.exit_code, 0, "Command should succeed") - self.assertIn("Config replaced successfully.", result.output) - - # Verify mocked_open was called as expected - mocked_open.assert_called_with(self.replace_file_path, 'r') - - def test_repalce_multiasic_missing_scope(self): - # Mock GenericUpdater to avoid actual patch application - with patch('config.main.GenericUpdater') as mock_generic_updater: - mock_generic_updater.return_value.replace = MagicMock() - - print("Multi ASIC: {}".format(multi_asic.is_multi_asic())) - # Invocation of the command with the CliRunner - result = self.runner.invoke(config.config.commands["replace"], - [self.replace_file_path], - catch_exceptions=True) - - print("Exit Code: {}, output: {}".format(result.exit_code, result.output)) - # Assertions and verifications - self.assertEqual(result.exit_code, 2, "Command should failed") - self.assertIn("Missing option \"-s\"", result.output) - - def test_repalce_multiasic_with_wrong_scope(self): - # Mock GenericUpdater to avoid actual patch application - with patch('config.main.GenericUpdater') as mock_generic_updater: - mock_generic_updater.return_value.replace = MagicMock() - - print("Multi ASIC: {}".format(multi_asic.is_multi_asic())) - # Invocation of the command with the CliRunner - result = self.runner.invoke(config.config.commands["replace"], - [self.replace_file_path, - "--scope", "x"], - catch_exceptions=True) - - print("Exit Code: {}, output: {}".format(result.exit_code, result.output)) - # Assertions and verifications - self.assertEqual(result.exit_code, 2, "Command should failed") - self.assertIn("Invalid value for \"-s\"", result.output) - - def test_checkpoint_multiasic(self): - checkpointname = "checkpointname" - # Mock GenericUpdater to avoid actual patch application - with patch('config.main.GenericUpdater') as mock_generic_updater: - mock_generic_updater.return_value.checkpoint = MagicMock() - - print("Multi ASIC: {}".format(multi_asic.is_multi_asic())) - # Invocation of the command with the CliRunner - result = self.runner.invoke(config.config.commands["checkpoint"], - [checkpointname], - catch_exceptions=True) - - print("Exit Code: {}, output: {}".format(result.exit_code, result.output)) - # Assertions and verifications - self.assertEqual(result.exit_code, 0, "Command should succeed") - self.assertIn("Checkpoint created successfully.", result.output) - - def test_rollback_multiasic(self): - checkpointname = "checkpointname" - # Mock GenericUpdater to avoid actual patch application - with patch('config.main.GenericUpdater') as mock_generic_updater: - mock_generic_updater.return_value.rollback = MagicMock() - - print("Multi ASIC: {}".format(multi_asic.is_multi_asic())) - # Invocation of the command with the CliRunner - result = self.runner.invoke(config.config.commands["rollback"], - [checkpointname], - catch_exceptions=True) - - print("Exit Code: {}, output: {}".format(result.exit_code, result.output)) - # Assertions and verifications - self.assertEqual(result.exit_code, 0, "Command should succeed") - self.assertIn("Config rolled back successfully.", result.output) - - def test_delete_checkpoint_multiasic(self): - checkpointname = "checkpointname" - # Mock GenericUpdater to avoid actual patch application - with patch('config.main.GenericUpdater') as mock_generic_updater: - mock_generic_updater.return_value.deletecheckpoint = MagicMock() - - print("Multi ASIC: {}".format(multi_asic.is_multi_asic())) - # Invocation of the command with the CliRunner - result = self.runner.invoke(config.config.commands["delete-checkpoint"], - [checkpointname], - catch_exceptions=True) - - print("Exit Code: {}, output: {}".format(result.exit_code, result.output)) - # Assertions and verifications - self.assertEqual(result.exit_code, 0, "Command should succeed") - self.assertIn("Checkpoint deleted successfully.", result.output) - @patch('config.main.subprocess.Popen') @patch('config.main.SonicYangCfgDbGenerator.validate_config_db_json', mock.Mock(return_value=True)) def test_apply_patch_validate_patch_multiasic(self, mock_subprocess_popen): @@ -2981,8 +2874,14 @@ def test_apply_patch_validate_patch_with_wrong_fetch_config(self, mock_subproces # Verify mocked_open was called as expected mocked_open.assert_called_with(self.replace_file_path, 'r') - - def test_repalce_multiasic(self): + + @patch('config.main.subprocess.Popen') + @patch('config.main.SonicYangCfgDbGenerator.validate_config_db_json', mock.Mock(return_value=True)) + def test_repalce_multiasic(self, mock_subprocess_popen): + mock_instance = MagicMock() + mock_instance.communicate.return_value = (json.dumps(self.all_config), 2) + mock_subprocess_popen.return_value = mock_instance + # Mock open to simulate file reading mock_replace_content = "[]" with patch('builtins.open', mock_open(read_data=json.dumps(mock_replace_content)), create=True) as mocked_open: @@ -2993,9 +2892,19 @@ def test_repalce_multiasic(self): print("Multi ASIC: {}".format(multi_asic.is_multi_asic())) # Invocation of the command with the CliRunner result = self.runner.invoke(config.config.commands["replace"], - [self.replace_file_path, - "--scope", "localhost"], - + [self.replace_file_path], + catch_exceptions=True) + + print("Exit Code: {}, output: {}".format(result.exit_code, result.output)) + # Assertions and verifications + self.assertEqual(result.exit_code, 0, "Command should succeed") + self.assertIn("Config replaced successfully.", result.output) + + # Verify mocked_open was called as expected + mocked_open.assert_called_with(self.replace_file_path, 'r') + + @patch('config.main.subprocess.Popen') + @patch('config.main.SonicYangCfgDbGenerator.validate_config_db_json', mock.Mock(return_value=True)) def test_repalce_multiasic_missing_scope(self): # Mock GenericUpdater to avoid actual patch application with patch('config.main.GenericUpdater') as mock_generic_updater: @@ -3012,6 +2921,8 @@ def test_repalce_multiasic_missing_scope(self): self.assertEqual(result.exit_code, 2, "Command should failed") self.assertIn("Missing option \"-s\"", result.output) + @patch('config.main.subprocess.Popen') + @patch('config.main.SonicYangCfgDbGenerator.validate_config_db_json', mock.Mock(return_value=True)) def test_repalce_multiasic_with_wrong_scope(self): # Mock GenericUpdater to avoid actual patch application with patch('config.main.GenericUpdater') as mock_generic_updater: @@ -3079,11 +2990,10 @@ def test_delete_checkpoint_multiasic(self): # Assertions and verifications self.assertEqual(result.exit_code, 0, "Command should succeed") self.assertIn("Checkpoint deleted successfully.", result.output) - self.assertNotEqual(result.exit_code, 0, "Command should failed.") - self.assertIn("Failed to apply patch", result.output) - - # Verify mocked_open was called as expected - mocked_open.assert_called_with(self.patch_file_path, 'r') + self.assertNotEqual(result.exit_code, 0, "Command should failed.") + self.assertIn("Failed to apply patch", result.output) + # Verify mocked_open was called as expected + mocked_open.assert_called_with(self.patch_file_path, 'r') @classmethod def teardown_class(cls): From c61344dc075bf3de0f720299759e0a67d363f343 Mon Sep 17 00:00:00 2001 From: Xincun Li Date: Fri, 17 May 2024 17:09:23 -0700 Subject: [PATCH 14/44] Fix pre-commit and replace function --- config/main.py | 7 +++-- tests/config_test.py | 70 +++++++++++++++----------------------------- 2 files changed, 28 insertions(+), 49 deletions(-) diff --git a/config/main.py b/config/main.py index e8e34981ef..b0ba76f27e 100644 --- a/config/main.py +++ b/config/main.py @@ -1499,10 +1499,11 @@ def replace(ctx, target_file_path, format, dry_run, ignore_non_yang_tables, igno scope_set = set(multi_asic.get_namespace_list() + [HOST_NAMESPACE]) missing_scopes = scope_set - config_keys if missing_scopes: - raise GenericConfigUpdaterError(f"To be replace config: {target_file_path} is missing these namespaces: {missing_scopes}") + raise GenericConfigUpdaterError(f"""To be replace config: {target_file_path} is missing + these namespaces: {missing_scopes}""") for scope in scope_set: - scope_config = target_config.pop("scope") + scope_config = target_config.pop(scope) if not SonicYangCfgDbGenerator().validate_config_db_json(scope_config): raise GenericConfigUpdaterError(f"Invalid config for {scope} in {target_file_path}") @@ -1515,7 +1516,7 @@ def replace(ctx, target_file_path, format, dry_run, ignore_non_yang_tables, igno raise GenericConfigUpdaterError(f"Invalid config in {target_file_path}") scope = multi_asic.DEFAULT_NAMESPACE GenericUpdater(namespace=scope).replace(target_config, config_format, verbose, dry_run, - ignore_non_yang_tables, ignore_path) + ignore_non_yang_tables, ignore_path) click.secho("Config replaced successfully.", fg="cyan", underline=True) except Exception as ex: diff --git a/tests/config_test.py b/tests/config_test.py index d4ba0fb9b8..858dfb70d6 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -2868,22 +2868,17 @@ def test_apply_patch_validate_patch_with_wrong_fetch_config(self, mock_subproces catch_exceptions=True) print("Exit Code: {}, output: {}".format(result.exit_code, result.output)) - # Assertions and verifications - self.assertEqual(result.exit_code, 0, "Command should succeed") - self.assertIn("Config replaced successfully.", result.output) + # Assertions and verifications + self.assertNotEqual(result.exit_code, 0, "Command should failed.") + self.assertIn("Failed to apply patch", result.output) # Verify mocked_open was called as expected - mocked_open.assert_called_with(self.replace_file_path, 'r') + mocked_open.assert_called_with(self.patch_file_path, 'r') - @patch('config.main.subprocess.Popen') @patch('config.main.SonicYangCfgDbGenerator.validate_config_db_json', mock.Mock(return_value=True)) - def test_repalce_multiasic(self, mock_subprocess_popen): - mock_instance = MagicMock() - mock_instance.communicate.return_value = (json.dumps(self.all_config), 2) - mock_subprocess_popen.return_value = mock_instance - + def test_repalce_multiasic(self): # Mock open to simulate file reading - mock_replace_content = "[]" + mock_replace_content = copy.deepcopy(self.all_config) with patch('builtins.open', mock_open(read_data=json.dumps(mock_replace_content)), create=True) as mocked_open: # Mock GenericUpdater to avoid actual patch application with patch('config.main.GenericUpdater') as mock_generic_updater: @@ -2903,42 +2898,29 @@ def test_repalce_multiasic(self, mock_subprocess_popen): # Verify mocked_open was called as expected mocked_open.assert_called_with(self.replace_file_path, 'r') - @patch('config.main.subprocess.Popen') @patch('config.main.SonicYangCfgDbGenerator.validate_config_db_json', mock.Mock(return_value=True)) def test_repalce_multiasic_missing_scope(self): - # Mock GenericUpdater to avoid actual patch application - with patch('config.main.GenericUpdater') as mock_generic_updater: - mock_generic_updater.return_value.replace = MagicMock() - - print("Multi ASIC: {}".format(multi_asic.is_multi_asic())) - # Invocation of the command with the CliRunner - result = self.runner.invoke(config.config.commands["replace"], - [self.replace_file_path], - catch_exceptions=True) - - print("Exit Code: {}, output: {}".format(result.exit_code, result.output)) - # Assertions and verifications - self.assertEqual(result.exit_code, 2, "Command should failed") - self.assertIn("Missing option \"-s\"", result.output) + # Mock open to simulate file reading + mock_replace_content = copy.deepcopy(self.all_config) + mock_replace_content.pop("asic0") + with patch('builtins.open', mock_open(read_data=json.dumps(mock_replace_content)), create=True) as mocked_open: + # Mock GenericUpdater to avoid actual patch application + with patch('config.main.GenericUpdater') as mock_generic_updater: + mock_generic_updater.return_value.replace = MagicMock() - @patch('config.main.subprocess.Popen') - @patch('config.main.SonicYangCfgDbGenerator.validate_config_db_json', mock.Mock(return_value=True)) - def test_repalce_multiasic_with_wrong_scope(self): - # Mock GenericUpdater to avoid actual patch application - with patch('config.main.GenericUpdater') as mock_generic_updater: - mock_generic_updater.return_value.replace = MagicMock() + print("Multi ASIC: {}".format(multi_asic.is_multi_asic())) + # Invocation of the command with the CliRunner + result = self.runner.invoke(config.config.commands["replace"], + [self.replace_file_path], + catch_exceptions=True) - print("Multi ASIC: {}".format(multi_asic.is_multi_asic())) - # Invocation of the command with the CliRunner - result = self.runner.invoke(config.config.commands["replace"], - [self.replace_file_path, - "--scope", "x"], - catch_exceptions=True) + print("Exit Code: {}, output: {}".format(result.exit_code, result.output)) + # Assertions and verifications + self.assertNotEqual(result.exit_code, 0, "Command should failed") + self.assertIn("Failed to replace config", result.output) - print("Exit Code: {}, output: {}".format(result.exit_code, result.output)) - # Assertions and verifications - self.assertEqual(result.exit_code, 2, "Command should failed") - self.assertIn("Invalid value for \"-s\"", result.output) + # Verify mocked_open was called as expected + mocked_open.assert_called_with(self.replace_file_path, 'r') def test_checkpoint_multiasic(self): checkpointname = "checkpointname" @@ -2990,10 +2972,6 @@ def test_delete_checkpoint_multiasic(self): # Assertions and verifications self.assertEqual(result.exit_code, 0, "Command should succeed") self.assertIn("Checkpoint deleted successfully.", result.output) - self.assertNotEqual(result.exit_code, 0, "Command should failed.") - self.assertIn("Failed to apply patch", result.output) - # Verify mocked_open was called as expected - mocked_open.assert_called_with(self.patch_file_path, 'r') @classmethod def teardown_class(cls): From 8016675a61f52112bde9b77a05e7fab9f2263fbe Mon Sep 17 00:00:00 2001 From: Xincun Li Date: Fri, 17 May 2024 17:17:06 -0700 Subject: [PATCH 15/44] Fix precommit --- config/main.py | 3 ++- tests/config_test.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/config/main.py b/config/main.py index b0ba76f27e..f22fe09bbe 100644 --- a/config/main.py +++ b/config/main.py @@ -1499,7 +1499,7 @@ def replace(ctx, target_file_path, format, dry_run, ignore_non_yang_tables, igno scope_set = set(multi_asic.get_namespace_list() + [HOST_NAMESPACE]) missing_scopes = scope_set - config_keys if missing_scopes: - raise GenericConfigUpdaterError(f"""To be replace config: {target_file_path} is missing + raise GenericConfigUpdaterError(f"""To be replace config: {target_file_path} is missing these namespaces: {missing_scopes}""") for scope in scope_set: @@ -1523,6 +1523,7 @@ def replace(ctx, target_file_path, format, dry_run, ignore_non_yang_tables, igno click.secho("Failed to replace config", fg="red", underline=True, err=True) ctx.fail(ex) + @config.command() @click.argument('checkpoint-name', type=str, required=True) @click.option('-d', '--dry-run', is_flag=True, default=False, help='test out the command without affecting config state') diff --git a/tests/config_test.py b/tests/config_test.py index 858dfb70d6..d1e8abbd02 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -2868,7 +2868,7 @@ def test_apply_patch_validate_patch_with_wrong_fetch_config(self, mock_subproces catch_exceptions=True) print("Exit Code: {}, output: {}".format(result.exit_code, result.output)) - # Assertions and verifications + # Assertions and verifications self.assertNotEqual(result.exit_code, 0, "Command should failed.") self.assertIn("Failed to apply patch", result.output) From 9c47e3831e0511210d5c024c2b7ff575cada6ae8 Mon Sep 17 00:00:00 2001 From: Xincun Li Date: Wed, 22 May 2024 16:51:00 -0700 Subject: [PATCH 16/44] Refactor checkpoint and rollback. --- config/main.py | 72 +++++----- generic_config_updater/generic_updater.py | 162 +++++++++++++++++++--- show/main.py | 23 +-- tests/config_test.py | 33 ++++- 4 files changed, 207 insertions(+), 83 deletions(-) diff --git a/config/main.py b/config/main.py index f22fe09bbe..8995861103 100644 --- a/config/main.py +++ b/config/main.py @@ -19,7 +19,7 @@ from jsonpatch import JsonPatchConflict from jsonpointer import JsonPointerException from collections import OrderedDict -from generic_config_updater.generic_updater import GenericUpdater, ConfigFormat, extract_scope +from generic_config_updater.generic_updater import GenericUpdater, ConfigFormat, MultiASICConfigRollbacker, extract_scope from generic_config_updater.gu_common import HOST_NAMESPACE, GenericConfigUpdaterError from minigraph import parse_device_desc_xml, minigraph_encoder from natsort import natsorted @@ -1482,10 +1482,7 @@ def replace(ctx, target_file_path, format, dry_run, ignore_non_yang_tables, igno **WARNING** The target config file should be the whole config, not just the part intended to be updated. - : Path to the target file on the file-system. - If the device is a Multi-ASIC environment, please give the namespace of specific ASIC, for instance: - localhost, asic0, asic1, ... - """ + : Path to the target file on the file-system.""" try: print_dry_run_message(dry_run) @@ -1494,25 +1491,19 @@ def replace(ctx, target_file_path, format, dry_run, ignore_non_yang_tables, igno target_config = json.loads(target_config_as_text) config_format = ConfigFormat[format.upper()] + if multi_asic.is_multi_asic(): - config_keys = set(target_config.keys()) - scope_set = set(multi_asic.get_namespace_list() + [HOST_NAMESPACE]) - missing_scopes = scope_set - config_keys - if missing_scopes: - raise GenericConfigUpdaterError(f"""To be replace config: {target_file_path} is missing - these namespaces: {missing_scopes}""") - - for scope in scope_set: - scope_config = target_config.pop(scope) + scope_list = [HOST_NAMESPACE] + scope_list.extend(multi_asic.get_namespace_list()) + tobevalidated = copy.deepcopy(target_config) + for scope in scope_list: + scope_config = tobevalidated.pop(scope) if not SonicYangCfgDbGenerator().validate_config_db_json(scope_config): raise GenericConfigUpdaterError(f"Invalid config for {scope} in {target_file_path}") - - if scope.lower() == HOST_NAMESPACE: - scope = multi_asic.DEFAULT_NAMESPACE - GenericUpdater(namespace=scope).replace(scope_config, config_format, verbose, dry_run, - ignore_non_yang_tables, ignore_path) + config_rollbacker = MultiASICConfigRollbacker(scopelist=scope_list) + config_rollbacker.replace_all(target_config) else: - if not SonicYangCfgDbGenerator().validate_config_db_json(scope_config): + if not SonicYangCfgDbGenerator().validate_config_db_json(target_config): raise GenericConfigUpdaterError(f"Invalid config in {target_file_path}") scope = multi_asic.DEFAULT_NAMESPACE GenericUpdater(namespace=scope).replace(target_config, config_format, verbose, dry_run, @@ -1539,12 +1530,14 @@ def rollback(ctx, checkpoint_name, dry_run, ignore_non_yang_tables, ignore_path, : The checkpoint name, use `config list-checkpoints` command to see available checkpoints.""" try: print_dry_run_message(dry_run) - asic_list = [multi_asic.DEFAULT_NAMESPACE] if multi_asic.is_multi_asic(): - asic_list.extend(multi_asic.get_namespace_list()) - for asic in asic_list: - GenericUpdater(namespace=asic).rollback(checkpoint_name, verbose, dry_run, - ignore_non_yang_tables, ignore_path) + scope_list = [multi_asic.DEFAULT_NAMESPACE] + scope_list.extend(multi_asic.get_namespace_list()) + config_rollbacker = MultiASICConfigRollbacker(scopelist=scope_list) + config_rollbacker.rollback_all(checkpoint_name) + else: + GenericUpdater(namespace=multi_asic.DEFAULT_NAMESPACE).rollback(checkpoint_name, verbose, dry_run, + ignore_non_yang_tables, ignore_path) click.secho("Config rolled back successfully.", fg="cyan", underline=True) except Exception as ex: @@ -1560,11 +1553,13 @@ def checkpoint(ctx, checkpoint_name, verbose): : The checkpoint name, use `config list-checkpoints` command to see available checkpoints.""" try: - asic_list = [multi_asic.DEFAULT_NAMESPACE] if multi_asic.is_multi_asic(): - asic_list.extend(multi_asic.get_namespace_list()) - for asic in asic_list: - GenericUpdater(namespace=asic).checkpoint(checkpoint_name, verbose) + scope_list = [multi_asic.DEFAULT_NAMESPACE] + scope_list.extend(multi_asic.get_namespace_list()) + config_rollbacker = MultiASICConfigRollbacker(scopelist=scope_list) + config_rollbacker.checkpoint_all(checkpoint_name) + else: + GenericUpdater(namespace=multi_asic.DEFAULT_NAMESPACE).checkpoint(checkpoint_name, verbose) click.secho("Checkpoint created successfully.", fg="cyan", underline=True) except Exception as ex: @@ -1580,11 +1575,13 @@ def delete_checkpoint(ctx, checkpoint_name, verbose): : The checkpoint name, use `config list-checkpoints` command to see available checkpoints.""" try: - asic_list = [multi_asic.DEFAULT_NAMESPACE] if multi_asic.is_multi_asic(): - asic_list.extend(multi_asic.get_namespace_list()) - for asic in asic_list: - GenericUpdater(namespace=asic).delete_checkpoint(checkpoint_name, verbose) + scope_list = [multi_asic.DEFAULT_NAMESPACE] + scope_list.extend(multi_asic.get_namespace_list()) + config_rollbacker = MultiASICConfigRollbacker(scopelist=scope_list) + config_rollbacker.delete_checkpoint(checkpoint_name) + else: + GenericUpdater(namespace=multi_asic.DEFAULT_NAMESPACE).delete_checkpoint(checkpoint_name, verbose) click.secho("Checkpoint deleted successfully.", fg="cyan", underline=True) except Exception as ex: @@ -1597,7 +1594,14 @@ def delete_checkpoint(ctx, checkpoint_name, verbose): def list_checkpoints(ctx, verbose): """List the config checkpoints available.""" try: - checkpoints_list = GenericUpdater().list_checkpoints(verbose) + if multi_asic.is_multi_asic(): + scope_list = [multi_asic.DEFAULT_NAMESPACE] + scope_list.extend(multi_asic.get_namespace_list()) + config_rollbacker = MultiASICConfigRollbacker(scopelist=scope_list) + checkpoints_list = config_rollbacker.list_checkpoints() + else: + checkpoints_list = GenericUpdater().list_checkpoints(verbose) + formatted_output = json.dumps(checkpoints_list, indent=4) click.echo(formatted_output) except Exception as ex: diff --git a/generic_config_updater/generic_updater.py b/generic_config_updater/generic_updater.py index 367eb97e53..60175750f3 100644 --- a/generic_config_updater/generic_updater.py +++ b/generic_config_updater/generic_updater.py @@ -1,4 +1,5 @@ import json +import subprocess import jsonpointer import os from enum import Enum @@ -36,6 +37,118 @@ def extract_scope(path): return scope, remainder +def get_cmd_output(cmd): + proc = subprocess.Popen(cmd, text=True, stdout=subprocess.PIPE) + return proc.communicate()[0], proc.returncode + + +def get_config_json_by_namespace(namespace): + cmd = ['sonic-cfggen', '-d', '--print-data'] + if namespace is not None and namespace != multi_asic.DEFAULT_NAMESPACE: + cmd += ['-n', namespace] + + stdout, rc = get_cmd_output(cmd) + if rc: + raise GenericConfigUpdaterError("Failed to get cmd output '{}':rc {}".format(cmd, rc)) + + try: + config_json = json.loads(stdout) + except json.JSONDecodeError as e: + raise GenericConfigUpdaterError("Failed to get config by '{}' due to {}".format(cmd, e)) + + return config_json + + +class MultiASICConfigRollbacker: + def __init__(self, scopelist, checkpoints_dir=CHECKPOINTS_DIR): + self.logger = genericUpdaterLogging.get_logger(title="MultiASICConfigRollbacker", + print_all_to_console=True) + self.scopelist = scopelist + self.checkpoints_dir = checkpoints_dir + self.util = Util(checkpoints_dir=checkpoints_dir) + + def replace_all(self, target_config): + config_keys = set(target_config.keys()) + missing_scopes = set(self.scopelist) - config_keys + if missing_scopes: + raise GenericConfigUpdaterError(f"To be replace config is missing scope: {missing_scopes}") + + for scope in self.scopelist: + scope_config = target_config.pop(scope) + if scope.lower() == HOST_NAMESPACE: + scope = multi_asic.DEFAULT_NAMESPACE + ConfigReplacer(namespace=scope).replace(scope_config) + + def rollback_all(self, checkpoint_name): + self.logger.log_notice("Config rollbacking starting.") + self.logger.log_notice(f"Checkpoint name: {checkpoint_name}.") + self.logger.log_notice(f"Verifying '{checkpoint_name}' exists.") + if not self.util.check_checkpoint_exists(checkpoint_name): + raise ValueError(f"Checkpoint '{checkpoint_name}' does not exist") + self.logger.log_notice(f"Loading checkpoint into memory.") + target_config = self.util.get_checkpoint_content(checkpoint_name) + self.logger.log_notice(f"Replacing config using 'Config Replacer'.") + for scope in self.scopelist: + if scope.lower() == multi_asic.DEFAULT_NAMESPACE: + config = target_config.pop(HOST_NAMESPACE) + else: + config = target_config.pop(scope) + ConfigReplacer(namespace=scope).replace(config) + self.logger.log_notice("Config rollbacking completed.") + + def checkpoint_all(self, checkpoint_name): + all_configs = {} + self.logger.log_notice("Config checkpoint starting.") + self.logger.log_notice(f"Checkpoint name: {checkpoint_name}.") + for scope in self.scopelist: + self.logger.log_notice(f"Getting current {scope} config db.") + config = get_config_json_by_namespace(scope) + if scope.lower() == multi_asic.DEFAULT_NAMESPACE: + scope = HOST_NAMESPACE + all_configs[scope] = config + + self.logger.log_notice("Getting checkpoint full-path.") + path = self.util.get_checkpoint_full_path(checkpoint_name) + self.logger.log_notice("Ensuring checkpoint directory exist.") + self.util.ensure_checkpoints_dir_exists() + self.logger.log_notice(f"Saving config db content to {path}.") + + self.util.save_json_file(path, all_configs) + self.logger.log_notice("Config checkpoint completed.") + + def list_checkpoints(self): + self.logger.log_info("Listing checkpoints starting.") + + self.logger.log_info(f"Verifying checkpoints directory '{self.checkpoints_dir}' exists.") + if not self.util.checkpoints_dir_exist(): + self.logger.log_info("Checkpoints directory is empty, returning empty checkpoints list.") + return [] + + self.logger.log_info("Getting checkpoints in checkpoints directory.") + checkpoint_names = self.util.get_checkpoint_names() + + checkpoints_len = len(checkpoint_names) + self.logger.log_info(f"Found {checkpoints_len} checkpoint{'s' if checkpoints_len != 1 else ''}{':' if checkpoints_len > 0 else '.'}") + for checkpoint_name in checkpoint_names: + self.logger.log_info(f" * {checkpoint_name}") + + self.logger.log_info("Listing checkpoints completed.") + + return checkpoint_names + + def delete_checkpoint(self, checkpoint_name): + self.logger.log_notice("Deleting checkpoint starting.") + self.logger.log_notice(f"Checkpoint name: {checkpoint_name}.") + + self.logger.log_notice(f"Checking checkpoint exists.") + if not self.util.check_checkpoint_exists(checkpoint_name): + raise ValueError(f"Checkpoint '{checkpoint_name}' does not exist") + + self.logger.log_notice(f"Deleting checkpoint.") + self.util.delete_checkpoint(checkpoint_name) + + self.logger.log_notice("Deleting checkpoint completed.") + class ConfigLock: def acquire_lock(self): # TODO: Implement ConfigLock @@ -160,21 +273,21 @@ def __init__(self, namespace=multi_asic.DEFAULT_NAMESPACE): self.namespace = namespace self.logger = genericUpdaterLogging.get_logger(title="Config Rollbacker", print_all_to_console=True) + self.util = Util(checkpoints_dir=checkpoints_dir) self.checkpoints_dir = checkpoints_dir self.config_replacer = config_replacer if config_replacer is not None else ConfigReplacer(namespace=self.namespace) self.config_wrapper = config_wrapper if config_wrapper is not None else ConfigWrapper(namespace=self.namespace) def rollback(self, checkpoint_name): - checkpoint_name = checkpoint_name + self.namespace self.logger.log_notice("Config rollbacking starting.") self.logger.log_notice(f"Checkpoint name: {checkpoint_name}.") self.logger.log_notice(f"Verifying '{checkpoint_name}' exists.") - if not self._check_checkpoint_exists(checkpoint_name): + if not self.util.check_checkpoint_exists(checkpoint_name): raise ValueError(f"Checkpoint '{checkpoint_name}' does not exist") self.logger.log_notice(f"Loading checkpoint into memory.") - target_config = self._get_checkpoint_content(checkpoint_name) + target_config = self.util.get_checkpoint_content(checkpoint_name) self.logger.log_notice(f"Replacing config using 'Config Replacer'.") self.config_replacer.replace(target_config) @@ -182,7 +295,6 @@ def rollback(self, checkpoint_name): self.logger.log_notice("Config rollbacking completed.") def checkpoint(self, checkpoint_name): - checkpoint_name = checkpoint_name + self.namespace self.logger.log_notice("Config checkpoint starting.") self.logger.log_notice(f"Checkpoint name: {checkpoint_name}.") @@ -190,13 +302,13 @@ def checkpoint(self, checkpoint_name): json_content = self.config_wrapper.get_config_db_as_json() self.logger.log_notice("Getting checkpoint full-path.") - path = self._get_checkpoint_full_path(checkpoint_name) + path = self.util.get_checkpoint_full_path(checkpoint_name) self.logger.log_notice("Ensuring checkpoint directory exist.") - self._ensure_checkpoints_dir_exists() + self.util.ensure_checkpoints_dir_exists() self.logger.log_notice(f"Saving config db content to {path}.") - self._save_json_file(path, json_content) + self.util.save_json_file(path, json_content) self.logger.log_notice("Config checkpoint completed.") @@ -204,12 +316,12 @@ def list_checkpoints(self): self.logger.log_info("Listing checkpoints starting.") self.logger.log_info(f"Verifying checkpoints directory '{self.checkpoints_dir}' exists.") - if not self._checkpoints_dir_exist(): + if not self.util.checkpoints_dir_exist(): self.logger.log_info("Checkpoints directory is empty, returning empty checkpoints list.") return [] self.logger.log_info("Getting checkpoints in checkpoints directory.") - checkpoint_names = self._get_checkpoint_names() + checkpoint_names = self.util.get_checkpoint_names() checkpoints_len = len(checkpoint_names) self.logger.log_info(f"Found {checkpoints_len} checkpoint{'s' if checkpoints_len != 1 else ''}{':' if checkpoints_len > 0 else '.'}") @@ -221,36 +333,40 @@ def list_checkpoints(self): return checkpoint_names def delete_checkpoint(self, checkpoint_name): - checkpoint_name = checkpoint_name + self.namespace self.logger.log_notice("Deleting checkpoint starting.") self.logger.log_notice(f"Checkpoint name: {checkpoint_name}.") self.logger.log_notice(f"Checking checkpoint exists.") - if not self._check_checkpoint_exists(checkpoint_name): + if not self.util.check_checkpoint_exists(checkpoint_name): raise ValueError(f"Checkpoint '{checkpoint_name}' does not exist") self.logger.log_notice(f"Deleting checkpoint.") - self._delete_checkpoint(checkpoint_name) + self.util.delete_checkpoint(checkpoint_name) self.logger.log_notice("Deleting checkpoint completed.") - def _ensure_checkpoints_dir_exists(self): + +class Util: + def __init__(self, checkpoints_dir=CHECKPOINTS_DIR): + self.checkpoints_dir = checkpoints_dir + + def ensure_checkpoints_dir_exists(self): os.makedirs(self.checkpoints_dir, exist_ok=True) - def _save_json_file(self, path, json_content): + def save_json_file(self, path, json_content): with open(path, "w") as fh: fh.write(json.dumps(json_content)) - def _get_checkpoint_content(self, checkpoint_name): - path = self._get_checkpoint_full_path(checkpoint_name) + def get_checkpoint_content(self, checkpoint_name): + path = self.get_checkpoint_full_path(checkpoint_name) with open(path) as fh: text = fh.read() return json.loads(text) - def _get_checkpoint_full_path(self, name): + def get_checkpoint_full_path(self, name): return os.path.join(self.checkpoints_dir, f"{name}{CHECKPOINT_EXT}") - def _get_checkpoint_names(self): + def get_checkpoint_names(self): file_names = [] for file_name in os.listdir(self.checkpoints_dir): if file_name.endswith(CHECKPOINT_EXT): @@ -260,15 +376,15 @@ def _get_checkpoint_names(self): return file_names - def _checkpoints_dir_exist(self): + def checkpoints_dir_exist(self): return os.path.isdir(self.checkpoints_dir) - def _check_checkpoint_exists(self, name): - path = self._get_checkpoint_full_path(name) + def check_checkpoint_exists(self, name): + path = self.get_checkpoint_full_path(name) return os.path.isfile(path) - def _delete_checkpoint(self, name): - path = self._get_checkpoint_full_path(name) + def delete_checkpoint(self, name): + path = self.get_checkpoint_full_path(name) return os.remove(path) diff --git a/show/main.py b/show/main.py index cfdf30d3c6..c970cd667f 100755 --- a/show/main.py +++ b/show/main.py @@ -5,6 +5,7 @@ import re import click +from generic_config_updater.generic_updater import get_config_json_by_namespace import lazy_object_proxy import utilities_common.cli as clicommon from sonic_py_common import multi_asic @@ -138,28 +139,6 @@ def run_command(command, display_cmd=False, return_cmd=False, shell=False): if rc != 0: sys.exit(rc) -def get_cmd_output(cmd): - proc = subprocess.Popen(cmd, text=True, stdout=subprocess.PIPE) - return proc.communicate()[0], proc.returncode - -def get_config_json_by_namespace(namespace): - cmd = ['sonic-cfggen', '-d', '--print-data'] - if namespace is not None and namespace != multi_asic.DEFAULT_NAMESPACE: - cmd += ['-n', namespace] - - stdout, rc = get_cmd_output(cmd) - if rc: - click.echo("Failed to get cmd output '{}':rc {}".format(cmd, rc)) - raise click.Abort() - - try: - config_json = json.loads(stdout) - except JSONDecodeError as e: - click.echo("Failed to load output '{}':{}".format(cmd, e)) - raise click.Abort() - - return config_json - # Lazy global class instance for SONiC interface name to alias conversion iface_alias_converter = lazy_object_proxy.Proxy(lambda: clicommon.InterfaceAliasConverter()) diff --git a/tests/config_test.py b/tests/config_test.py index d1e8abbd02..a9cc33a601 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -1212,6 +1212,7 @@ def test_replace__help__gets_help_msg(self): self.assertEqual(expected_exit_code, result.exit_code) self.assertTrue(expected_output in result.output) + @patch('config.main.SonicYangCfgDbGenerator.validate_config_db_json', mock.Mock(return_value=True)) def test_replace__only_required_params__default_values_used_for_optional_params(self): # Arrange expected_exit_code = 0 @@ -1230,6 +1231,7 @@ def test_replace__only_required_params__default_values_used_for_optional_params( mock_generic_updater.replace.assert_called_once() mock_generic_updater.replace.assert_has_calls([expected_call_with_default_values]) + @patch('config.main.SonicYangCfgDbGenerator.validate_config_db_json', mock.Mock(return_value=True)) def test_replace__all_optional_params_non_default__non_default_values_used(self): # Arrange expected_exit_code = 0 @@ -1259,6 +1261,7 @@ def test_replace__all_optional_params_non_default__non_default_values_used(self) mock_generic_updater.replace.assert_called_once() mock_generic_updater.replace.assert_has_calls([expected_call_with_non_default_values]) + @patch('config.main.SonicYangCfgDbGenerator.validate_config_db_json', mock.Mock(return_value=True)) def test_replace__exception_thrown__error_displayed_error_code_returned(self): # Arrange unexpected_exit_code = 0 @@ -1277,6 +1280,7 @@ def test_replace__exception_thrown__error_displayed_error_code_returned(self): self.assertNotEqual(unexpected_exit_code, result.exit_code) self.assertTrue(any_error_message in result.output) + @patch('config.main.SonicYangCfgDbGenerator.validate_config_db_json', mock.Mock(return_value=True)) def test_replace__optional_parameters_passed_correctly(self): self.validate_replace_optional_parameter( ["--format", ConfigFormat.SONICYANG.name], @@ -2881,8 +2885,8 @@ def test_repalce_multiasic(self): mock_replace_content = copy.deepcopy(self.all_config) with patch('builtins.open', mock_open(read_data=json.dumps(mock_replace_content)), create=True) as mocked_open: # Mock GenericUpdater to avoid actual patch application - with patch('config.main.GenericUpdater') as mock_generic_updater: - mock_generic_updater.return_value.replace = MagicMock() + with patch('config.main.MultiASICConfigRollbacker') as mock_generic_updater: + mock_generic_updater.return_value.replace_all = MagicMock() print("Multi ASIC: {}".format(multi_asic.is_multi_asic())) # Invocation of the command with the CliRunner @@ -2922,7 +2926,22 @@ def test_repalce_multiasic_missing_scope(self): # Verify mocked_open was called as expected mocked_open.assert_called_with(self.replace_file_path, 'r') - def test_checkpoint_multiasic(self): + @patch('generic_config_updater.generic_updater.subprocess.Popen') + @patch('generic_config_updater.generic_updater.Util.ensure_checkpoints_dir_exists', mock.Mock(return_value=True)) + @patch('generic_config_updater.generic_updater.Util.save_json_file', MagicMock()) + def test_checkpoint_multiasic(self, mock_subprocess_popen): + allconfigs = copy.deepcopy(self.all_config) + side_effects = [ + (json.dumps(allconfigs.pop("localhost")), 0), + (json.dumps(allconfigs.pop("asic0")), 0), + (json.dumps(allconfigs.pop("asic1")), 0) + ] + + mock_instance = MagicMock() + mock_instance.communicate.side_effect = side_effects + mock_instance.returncode = 0 + mock_subprocess_popen.return_value = mock_instance + checkpointname = "checkpointname" # Mock GenericUpdater to avoid actual patch application with patch('config.main.GenericUpdater') as mock_generic_updater: @@ -2939,7 +2958,11 @@ def test_checkpoint_multiasic(self): self.assertEqual(result.exit_code, 0, "Command should succeed") self.assertIn("Checkpoint created successfully.", result.output) - def test_rollback_multiasic(self): + @patch('generic_config_updater.generic_updater.ConfigReplacer.replace', MagicMock()) + @patch('generic_config_updater.generic_updater.Util.check_checkpoint_exists', mock.Mock(return_value=True)) + @patch('generic_config_updater.generic_updater.Util.get_checkpoint_content') + def test_rollback_multiasic(self, mock_get_checkpoint_content): + mock_get_checkpoint_content.return_value = copy.deepcopy(self.all_config) checkpointname = "checkpointname" # Mock GenericUpdater to avoid actual patch application with patch('config.main.GenericUpdater') as mock_generic_updater: @@ -2956,6 +2979,8 @@ def test_rollback_multiasic(self): self.assertEqual(result.exit_code, 0, "Command should succeed") self.assertIn("Config rolled back successfully.", result.output) + @patch('generic_config_updater.generic_updater.Util.delete_checkpoint', MagicMock()) + @patch('generic_config_updater.generic_updater.Util.check_checkpoint_exists', mock.Mock(return_value=True)) def test_delete_checkpoint_multiasic(self): checkpointname = "checkpointname" # Mock GenericUpdater to avoid actual patch application From e257b786245f5b73c98af7626c33a60d208424d9 Mon Sep 17 00:00:00 2001 From: Xincun Li Date: Wed, 22 May 2024 17:01:31 -0700 Subject: [PATCH 17/44] fix format --- config/main.py | 3 ++- generic_config_updater/generic_updater.py | 13 +++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/config/main.py b/config/main.py index 8995861103..68ccd127e8 100644 --- a/config/main.py +++ b/config/main.py @@ -19,7 +19,8 @@ from jsonpatch import JsonPatchConflict from jsonpointer import JsonPointerException from collections import OrderedDict -from generic_config_updater.generic_updater import GenericUpdater, ConfigFormat, MultiASICConfigRollbacker, extract_scope +from generic_config_updater.generic_updater import GenericUpdater, ConfigFormat +from generic_config_updater.generic_updater import MultiASICConfigRollbacker, extract_scope from generic_config_updater.gu_common import HOST_NAMESPACE, GenericConfigUpdaterError from minigraph import parse_device_desc_xml, minigraph_encoder from natsort import natsorted diff --git a/generic_config_updater/generic_updater.py b/generic_config_updater/generic_updater.py index 60175750f3..371961feeb 100644 --- a/generic_config_updater/generic_updater.py +++ b/generic_config_updater/generic_updater.py @@ -85,9 +85,9 @@ def rollback_all(self, checkpoint_name): self.logger.log_notice(f"Verifying '{checkpoint_name}' exists.") if not self.util.check_checkpoint_exists(checkpoint_name): raise ValueError(f"Checkpoint '{checkpoint_name}' does not exist") - self.logger.log_notice(f"Loading checkpoint into memory.") + self.logger.log_notice(f"Loading checkpoint '{checkpoint_name}' into memory.") target_config = self.util.get_checkpoint_content(checkpoint_name) - self.logger.log_notice(f"Replacing config using 'Config Replacer'.") + self.logger.log_notice(f"Replacing config '{checkpoint_name}' using 'Config Replacer'.") for scope in self.scopelist: if scope.lower() == multi_asic.DEFAULT_NAMESPACE: config = target_config.pop(HOST_NAMESPACE) @@ -128,7 +128,8 @@ def list_checkpoints(self): checkpoint_names = self.util.get_checkpoint_names() checkpoints_len = len(checkpoint_names) - self.logger.log_info(f"Found {checkpoints_len} checkpoint{'s' if checkpoints_len != 1 else ''}{':' if checkpoints_len > 0 else '.'}") + self.logger.log_info( + f"Found {checkpoints_len} checkpoint{'s' if checkpoints_len != 1 else ''}{':' if checkpoints_len > 0 else '.'}") for checkpoint_name in checkpoint_names: self.logger.log_info(f" * {checkpoint_name}") @@ -140,14 +141,14 @@ def delete_checkpoint(self, checkpoint_name): self.logger.log_notice("Deleting checkpoint starting.") self.logger.log_notice(f"Checkpoint name: {checkpoint_name}.") - self.logger.log_notice(f"Checking checkpoint exists.") + self.logger.log_notice(f"Checking checkpoint: {checkpoint_name} exists.") if not self.util.check_checkpoint_exists(checkpoint_name): raise ValueError(f"Checkpoint '{checkpoint_name}' does not exist") - self.logger.log_notice(f"Deleting checkpoint.") + self.logger.log_notice(f"Deleting checkpoint: {checkpoint_name}.") self.util.delete_checkpoint(checkpoint_name) - self.logger.log_notice("Deleting checkpoint completed.") + self.logger.log_notice(f"Deleting checkpoint: {checkpoint_name} completed.") class ConfigLock: def acquire_lock(self): From 1cd4b88f793c864bfab595af155c1c6da9da65e4 Mon Sep 17 00:00:00 2001 From: Xincun Li Date: Wed, 22 May 2024 17:04:42 -0700 Subject: [PATCH 18/44] Fix format --- generic_config_updater/generic_updater.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/generic_config_updater/generic_updater.py b/generic_config_updater/generic_updater.py index 371961feeb..4e6281eacf 100644 --- a/generic_config_updater/generic_updater.py +++ b/generic_config_updater/generic_updater.py @@ -129,7 +129,9 @@ def list_checkpoints(self): checkpoints_len = len(checkpoint_names) self.logger.log_info( - f"Found {checkpoints_len} checkpoint{'s' if checkpoints_len != 1 else ''}{':' if checkpoints_len > 0 else '.'}") + f"Found {checkpoints_len} checkpoint{'s' if checkpoints_len != 1 else ''}" + f"{':' if checkpoints_len > 0 else '.'}" + ) for checkpoint_name in checkpoint_names: self.logger.log_info(f" * {checkpoint_name}") From 87760a1e5bbe1fc283af51562fda9a824c6f2b16 Mon Sep 17 00:00:00 2001 From: Xincun Li Date: Wed, 22 May 2024 17:58:31 -0700 Subject: [PATCH 19/44] Fix UT --- tests/show_test.py | 45 +++++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/tests/show_test.py b/tests/show_test.py index 4cd29ac45e..ce9e82c531 100644 --- a/tests/show_test.py +++ b/tests/show_test.py @@ -41,32 +41,33 @@ def setup_class(cls): def mock_run_bgp_command(): return "" - def test_show_runningconfiguration_all_json_loads_failure(self): - def get_cmd_output_side_effect(*args, **kwargs): - return "", 0 - with mock.patch('show.main.get_cmd_output', - mock.MagicMock(side_effect=get_cmd_output_side_effect)) as mock_get_cmd_output: - result = CliRunner().invoke(show.cli.commands['runningconfiguration'].commands['all'], []) + @patch('generic_config_updater.generic_updater.subprocess.Popen') + def test_show_runningconfiguration_all_json_loads_failure(self, mock_subprocess_popen): + mock_instance = MagicMock() + mock_instance.communicate.return_value = ("", 2) + mock_subprocess_popen.return_value = mock_instance + result = CliRunner().invoke(show.cli.commands['runningconfiguration'].commands['all'], []) assert result.exit_code != 0 - def test_show_runningconfiguration_all_get_cmd_ouput_failure(self): - def get_cmd_output_side_effect(*args, **kwargs): - return "{}", 2 - with mock.patch('show.main.get_cmd_output', - mock.MagicMock(side_effect=get_cmd_output_side_effect)) as mock_get_cmd_output: - result = CliRunner().invoke(show.cli.commands['runningconfiguration'].commands['all'], []) + @patch('generic_config_updater.generic_updater.subprocess.Popen') + def test_show_runningconfiguration_all_get_cmd_ouput_failure(self, mock_subprocess_popen): + mock_instance = MagicMock() + mock_instance.communicate.return_value = ("{}", 2) + mock_subprocess_popen.return_value = mock_instance + result = CliRunner().invoke(show.cli.commands['runningconfiguration'].commands['all'], []) assert result.exit_code != 0 - def test_show_runningconfiguration_all(self): - def get_cmd_output_side_effect(*args, **kwargs): - return "{}", 0 - with mock.patch('show.main.get_cmd_output', - mock.MagicMock(side_effect=get_cmd_output_side_effect)) as mock_get_cmd_output: - result = CliRunner().invoke(show.cli.commands['runningconfiguration'].commands['all'], []) + @patch('generic_config_updater.generic_updater.subprocess.Popen') + def test_show_runningconfiguration_all(self, mock_subprocess_popen): + mock_instance = MagicMock() + mock_instance.communicate.return_value = ("{}", 0) + mock_subprocess_popen.return_value = mock_instance + mock_instance.returncode = 0 + result = CliRunner().invoke(show.cli.commands['runningconfiguration'].commands['all'], []) assert result.exit_code == 0 - assert mock_get_cmd_output.call_count == 1 - assert mock_get_cmd_output.call_args_list == [ - call(['sonic-cfggen', '-d', '--print-data'])] + assert mock_subprocess_popen.call_count == 1 + assert mock_subprocess_popen.call_args_list == [ + call(['sonic-cfggen', '-d', '--print-data'], text=True, stdout=-1)] @classmethod def teardown_class(cls): @@ -97,7 +98,7 @@ def mock_run_bgp_command(): def test_show_runningconfiguration_all_masic(self): def get_cmd_output_side_effect(*args, **kwargs): return "{}", 0 - with mock.patch('show.main.get_cmd_output', + with mock.patch('generic_config_updater.generic_updater.get_cmd_output', mock.MagicMock(side_effect=get_cmd_output_side_effect)) as mock_get_cmd_output: result = CliRunner().invoke(show.cli.commands['runningconfiguration'].commands['all'], []) assert result.exit_code == 0 From cf2f399b1e9e7bc698c75e941c8c644aeefe7feb Mon Sep 17 00:00:00 2001 From: xincunli-sonic Date: Fri, 24 May 2024 14:25:36 -0700 Subject: [PATCH 20/44] Fix UT. --- tests/config_test.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/config_test.py b/tests/config_test.py index 02db0aa83a..589b5ba531 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -3089,6 +3089,7 @@ def test_apply_patch_validate_patch_with_wrong_fetch_config(self, mock_subproces mocked_open.assert_called_with(self.patch_file_path, 'r') @patch('config.main.SonicYangCfgDbGenerator.validate_config_db_json', mock.Mock(return_value=True)) + @patch('generic_config_updater.generic_updater.ConfigReplacer.replace', MagicMock()) def test_repalce_multiasic(self): # Mock open to simulate file reading mock_replace_content = copy.deepcopy(self.all_config) @@ -3112,6 +3113,7 @@ def test_repalce_multiasic(self): mocked_open.assert_called_with(self.replace_file_path, 'r') @patch('config.main.SonicYangCfgDbGenerator.validate_config_db_json', mock.Mock(return_value=True)) + @patch('generic_config_updater.generic_updater.ConfigReplacer.replace', MagicMock()) def test_repalce_multiasic_missing_scope(self): # Mock open to simulate file reading mock_replace_content = copy.deepcopy(self.all_config) @@ -3174,8 +3176,8 @@ def test_rollback_multiasic(self, mock_get_checkpoint_content): mock_get_checkpoint_content.return_value = copy.deepcopy(self.all_config) checkpointname = "checkpointname" # Mock GenericUpdater to avoid actual patch application - with patch('config.main.GenericUpdater') as mock_generic_updater: - mock_generic_updater.return_value.rollback = MagicMock() + with patch('config.main.MultiASICConfigRollbacker') as mock_generic_updater: + mock_generic_updater.return_value.rollback_all = MagicMock() print("Multi ASIC: {}".format(multi_asic.is_multi_asic())) # Invocation of the command with the CliRunner From 57e9d3ab3606fa629b56c082464437e6ba698704 Mon Sep 17 00:00:00 2001 From: xincunli-sonic Date: Fri, 24 May 2024 15:53:55 -0700 Subject: [PATCH 21/44] Add UT. --- tests/config_test.py | 42 ++++++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/tests/config_test.py b/tests/config_test.py index 589b5ba531..b68b143090 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -3175,28 +3175,38 @@ def test_checkpoint_multiasic(self, mock_subprocess_popen): def test_rollback_multiasic(self, mock_get_checkpoint_content): mock_get_checkpoint_content.return_value = copy.deepcopy(self.all_config) checkpointname = "checkpointname" - # Mock GenericUpdater to avoid actual patch application - with patch('config.main.MultiASICConfigRollbacker') as mock_generic_updater: - mock_generic_updater.return_value.rollback_all = MagicMock() - - print("Multi ASIC: {}".format(multi_asic.is_multi_asic())) - # Invocation of the command with the CliRunner - result = self.runner.invoke(config.config.commands["rollback"], - [checkpointname], - catch_exceptions=True) - - print("Exit Code: {}, output: {}".format(result.exit_code, result.output)) - # Assertions and verifications - self.assertEqual(result.exit_code, 0, "Command should succeed") - self.assertIn("Config rolled back successfully.", result.output) + print("Multi ASIC: {}".format(multi_asic.is_multi_asic())) + # Invocation of the command with the CliRunner + result = self.runner.invoke(config.config.commands["rollback"], + [checkpointname], + catch_exceptions=True) + + print("Exit Code: {}, output: {}".format(result.exit_code, result.output)) + # Assertions and verifications + self.assertEqual(result.exit_code, 0, "Command should succeed") + self.assertIn("Config rolled back successfully.", result.output) + + + @patch('generic_config_updater.generic_updater.Util.checkpoints_dir_exist', mock.Mock(return_value=True)) + @patch('generic_config_updater.generic_updater.Util.get_checkpoint_names', mock.Mock(return_value=["checkpointname"])) + def test_list_checkpoint_multiasic(self): + print("Multi ASIC: {}".format(multi_asic.is_multi_asic())) + # Invocation of the command with the CliRunner + result = self.runner.invoke(config.config.commands["list-checkpoints"], + catch_exceptions=True) + + print("Exit Code: {}, output: {}".format(result.exit_code, result.output)) + # Assertions and verifications + self.assertEqual(result.exit_code, 0, "Command should succeed") + self.assertIn("checkpointname", result.output) @patch('generic_config_updater.generic_updater.Util.delete_checkpoint', MagicMock()) @patch('generic_config_updater.generic_updater.Util.check_checkpoint_exists', mock.Mock(return_value=True)) def test_delete_checkpoint_multiasic(self): checkpointname = "checkpointname" # Mock GenericUpdater to avoid actual patch application - with patch('config.main.GenericUpdater') as mock_generic_updater: - mock_generic_updater.return_value.deletecheckpoint = MagicMock() + with patch('config.main.MultiASICConfigRollbacker') as mock_generic_updater: + mock_generic_updater.return_value.delete_checkpoint = MagicMock() print("Multi ASIC: {}".format(multi_asic.is_multi_asic())) # Invocation of the command with the CliRunner From 4716f4d0dc94d402e589d8f357d58d35ff286382 Mon Sep 17 00:00:00 2001 From: xincunli-sonic Date: Fri, 24 May 2024 15:56:43 -0700 Subject: [PATCH 22/44] Fix format. --- tests/config_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/config_test.py b/tests/config_test.py index b68b143090..5a237a70d5 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -3186,9 +3186,9 @@ def test_rollback_multiasic(self, mock_get_checkpoint_content): self.assertEqual(result.exit_code, 0, "Command should succeed") self.assertIn("Config rolled back successfully.", result.output) - @patch('generic_config_updater.generic_updater.Util.checkpoints_dir_exist', mock.Mock(return_value=True)) - @patch('generic_config_updater.generic_updater.Util.get_checkpoint_names', mock.Mock(return_value=["checkpointname"])) + @patch('generic_config_updater.generic_updater.Util.get_checkpoint_names', + mock.Mock(return_value=["checkpointname"])) def test_list_checkpoint_multiasic(self): print("Multi ASIC: {}".format(multi_asic.is_multi_asic())) # Invocation of the command with the CliRunner From 1bfb020b48a2a21473134f2b3b50fdfe1ec17a9f Mon Sep 17 00:00:00 2001 From: xincunli-sonic Date: Fri, 24 May 2024 22:16:56 -0700 Subject: [PATCH 23/44] Fix UT --- tests/config_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/config_test.py b/tests/config_test.py index 5a237a70d5..752c3b98e9 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -3169,8 +3169,8 @@ def test_checkpoint_multiasic(self, mock_subprocess_popen): self.assertEqual(result.exit_code, 0, "Command should succeed") self.assertIn("Checkpoint created successfully.", result.output) - @patch('generic_config_updater.generic_updater.ConfigReplacer.replace', MagicMock()) @patch('generic_config_updater.generic_updater.Util.check_checkpoint_exists', mock.Mock(return_value=True)) + @patch('generic_config_updater.generic_updater.ConfigReplacer.replace', MagicMock()) @patch('generic_config_updater.generic_updater.Util.get_checkpoint_content') def test_rollback_multiasic(self, mock_get_checkpoint_content): mock_get_checkpoint_content.return_value = copy.deepcopy(self.all_config) From bcdfb8fc81f4831e4b82783caafa070428f33c68 Mon Sep 17 00:00:00 2001 From: xincunli-sonic Date: Sat, 25 May 2024 15:14:04 -0700 Subject: [PATCH 24/44] Fix UT. --- tests/generic_config_updater/change_applier_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/generic_config_updater/change_applier_test.py b/tests/generic_config_updater/change_applier_test.py index 4c9b33c3a4..dbdd4992f3 100644 --- a/tests/generic_config_updater/change_applier_test.py +++ b/tests/generic_config_updater/change_applier_test.py @@ -242,6 +242,7 @@ def test_change_apply(self, mock_set, mock_db, mock_subprocess_Popen): running_config = copy.deepcopy(read_data["running_data"]) json_changes = copy.deepcopy(read_data["json_changes"]) + generic_config_updater.change_applier.ChangeApplier.updater_conf = None generic_config_updater.change_applier.UPDATER_CONF_FILE = CONF_FILE generic_config_updater.change_applier.set_verbose(True) generic_config_updater.services_validator.set_verbose(True) From c8371eb711c95ca8d6b366e65e7af993f70d7c3d Mon Sep 17 00:00:00 2001 From: Xincun Li Date: Wed, 29 May 2024 11:20:30 -0700 Subject: [PATCH 25/44] Refactor replacer and rollbacker --- config/main.py | 57 +- generic_config_updater/change_applier.py | 16 +- generic_config_updater/generic_updater.py | 1062 ++++++++++----------- generic_config_updater/gu_common.py | 20 +- tests/config_test.py | 21 +- 5 files changed, 547 insertions(+), 629 deletions(-) diff --git a/config/main.py b/config/main.py index 68ccd127e8..f30951466e 100644 --- a/config/main.py +++ b/config/main.py @@ -19,8 +19,7 @@ from jsonpatch import JsonPatchConflict from jsonpointer import JsonPointerException from collections import OrderedDict -from generic_config_updater.generic_updater import GenericUpdater, ConfigFormat -from generic_config_updater.generic_updater import MultiASICConfigRollbacker, extract_scope +from generic_config_updater.generic_updater import GenericUpdater, ConfigFormat, extract_scope from generic_config_updater.gu_common import HOST_NAMESPACE, GenericConfigUpdaterError from minigraph import parse_device_desc_xml, minigraph_encoder from natsort import natsorted @@ -1169,7 +1168,7 @@ def apply_patch_for_scope(scope_changes, results, config_format, verbose, dry_ru scope_for_log = scope if scope else HOST_NAMESPACE try: # Call apply_patch with the ASIC-specific changes and predefined parameters - GenericUpdater(namespace=scope).apply_patch(jsonpatch.JsonPatch(changes), + GenericUpdater(scope=scope).apply_patch(jsonpatch.JsonPatch(changes), config_format, verbose, dry_run, @@ -1493,22 +1492,7 @@ def replace(ctx, target_file_path, format, dry_run, ignore_non_yang_tables, igno config_format = ConfigFormat[format.upper()] - if multi_asic.is_multi_asic(): - scope_list = [HOST_NAMESPACE] - scope_list.extend(multi_asic.get_namespace_list()) - tobevalidated = copy.deepcopy(target_config) - for scope in scope_list: - scope_config = tobevalidated.pop(scope) - if not SonicYangCfgDbGenerator().validate_config_db_json(scope_config): - raise GenericConfigUpdaterError(f"Invalid config for {scope} in {target_file_path}") - config_rollbacker = MultiASICConfigRollbacker(scopelist=scope_list) - config_rollbacker.replace_all(target_config) - else: - if not SonicYangCfgDbGenerator().validate_config_db_json(target_config): - raise GenericConfigUpdaterError(f"Invalid config in {target_file_path}") - scope = multi_asic.DEFAULT_NAMESPACE - GenericUpdater(namespace=scope).replace(target_config, config_format, verbose, dry_run, - ignore_non_yang_tables, ignore_path) + GenericUpdater().replace(target_config, config_format, verbose, dry_run, ignore_non_yang_tables, ignore_path) click.secho("Config replaced successfully.", fg="cyan", underline=True) except Exception as ex: @@ -1531,14 +1515,8 @@ def rollback(ctx, checkpoint_name, dry_run, ignore_non_yang_tables, ignore_path, : The checkpoint name, use `config list-checkpoints` command to see available checkpoints.""" try: print_dry_run_message(dry_run) - if multi_asic.is_multi_asic(): - scope_list = [multi_asic.DEFAULT_NAMESPACE] - scope_list.extend(multi_asic.get_namespace_list()) - config_rollbacker = MultiASICConfigRollbacker(scopelist=scope_list) - config_rollbacker.rollback_all(checkpoint_name) - else: - GenericUpdater(namespace=multi_asic.DEFAULT_NAMESPACE).rollback(checkpoint_name, verbose, dry_run, - ignore_non_yang_tables, ignore_path) + + GenericUpdater().rollback(checkpoint_name, verbose, dry_run, ignore_non_yang_tables, ignore_path) click.secho("Config rolled back successfully.", fg="cyan", underline=True) except Exception as ex: @@ -1554,13 +1532,7 @@ def checkpoint(ctx, checkpoint_name, verbose): : The checkpoint name, use `config list-checkpoints` command to see available checkpoints.""" try: - if multi_asic.is_multi_asic(): - scope_list = [multi_asic.DEFAULT_NAMESPACE] - scope_list.extend(multi_asic.get_namespace_list()) - config_rollbacker = MultiASICConfigRollbacker(scopelist=scope_list) - config_rollbacker.checkpoint_all(checkpoint_name) - else: - GenericUpdater(namespace=multi_asic.DEFAULT_NAMESPACE).checkpoint(checkpoint_name, verbose) + GenericUpdater().checkpoint(checkpoint_name, verbose) click.secho("Checkpoint created successfully.", fg="cyan", underline=True) except Exception as ex: @@ -1576,13 +1548,7 @@ def delete_checkpoint(ctx, checkpoint_name, verbose): : The checkpoint name, use `config list-checkpoints` command to see available checkpoints.""" try: - if multi_asic.is_multi_asic(): - scope_list = [multi_asic.DEFAULT_NAMESPACE] - scope_list.extend(multi_asic.get_namespace_list()) - config_rollbacker = MultiASICConfigRollbacker(scopelist=scope_list) - config_rollbacker.delete_checkpoint(checkpoint_name) - else: - GenericUpdater(namespace=multi_asic.DEFAULT_NAMESPACE).delete_checkpoint(checkpoint_name, verbose) + GenericUpdater().delete_checkpoint(checkpoint_name, verbose) click.secho("Checkpoint deleted successfully.", fg="cyan", underline=True) except Exception as ex: @@ -1595,14 +1561,7 @@ def delete_checkpoint(ctx, checkpoint_name, verbose): def list_checkpoints(ctx, verbose): """List the config checkpoints available.""" try: - if multi_asic.is_multi_asic(): - scope_list = [multi_asic.DEFAULT_NAMESPACE] - scope_list.extend(multi_asic.get_namespace_list()) - config_rollbacker = MultiASICConfigRollbacker(scopelist=scope_list) - checkpoints_list = config_rollbacker.list_checkpoints() - else: - checkpoints_list = GenericUpdater().list_checkpoints(verbose) - + checkpoints_list = GenericUpdater().list_checkpoints(verbose) formatted_output = json.dumps(checkpoints_list, indent=4) click.echo(formatted_output) except Exception as ex: diff --git a/generic_config_updater/change_applier.py b/generic_config_updater/change_applier.py index 32a356bf9a..74ce2353fe 100644 --- a/generic_config_updater/change_applier.py +++ b/generic_config_updater/change_applier.py @@ -34,8 +34,8 @@ def log_error(m): logger.log(logger.LOG_PRIORITY_ERROR, m, print_to_console) -def get_config_db(namespace=multi_asic.DEFAULT_NAMESPACE): - config_db = ConfigDBConnector(use_unix_socket_path=True, namespace=namespace) +def get_config_db(scope=multi_asic.DEFAULT_NAMESPACE): + config_db = ConfigDBConnector(use_unix_socket_path=True, namespace=scope) config_db.connect() return config_db @@ -74,9 +74,9 @@ class ChangeApplier: updater_conf = None - def __init__(self, namespace=multi_asic.DEFAULT_NAMESPACE): - self.namespace = namespace - self.config_db = get_config_db(self.namespace) + def __init__(self, scope=multi_asic.DEFAULT_NAMESPACE): + self.scope = scope + self.config_db = get_config_db(self.scope) self.backend_tables = [ "BUFFER_PG", "BUFFER_PROFILE", @@ -169,8 +169,8 @@ def remove_backend_tables_from_config(self, data): def _get_running_config(self): _, fname = tempfile.mkstemp(suffix="_changeApplier") - if self.namespace: - cmd = ['sonic-cfggen', '-d', '--print-data', '-n', self.namespace] + if self.scope: + cmd = ['sonic-cfggen', '-d', '--print-data', '-n', self.scope] else: cmd = ['sonic-cfggen', '-d', '--print-data'] @@ -181,7 +181,7 @@ def _get_running_config(self): return_code = result.returncode if return_code: os.remove(fname) - raise GenericConfigUpdaterError(f"Failed to get running config for namespace: {self.namespace}, Return code: {return_code}, Error: {err}") + raise GenericConfigUpdaterError(f"Failed to get running config for scope: {self.scope}, Return code: {return_code}, Error: {err}") run_data = {} try: diff --git a/generic_config_updater/generic_updater.py b/generic_config_updater/generic_updater.py index 4e6281eacf..7ca9acdc99 100644 --- a/generic_config_updater/generic_updater.py +++ b/generic_config_updater/generic_updater.py @@ -1,609 +1,587 @@ -import json -import subprocess -import jsonpointer -import os -from enum import Enum -from .gu_common import HOST_NAMESPACE, GenericConfigUpdaterError, EmptyTableError, ConfigWrapper, \ - DryRunConfigWrapper, PatchWrapper, genericUpdaterLogging -from .patch_sorter import StrictPatchSorter, NonStrictPatchSorter, ConfigSplitter, \ - TablesWithoutYangConfigSplitter, IgnorePathsFromYangConfigSplitter -from .change_applier import ChangeApplier, DryRunChangeApplier -from sonic_py_common import multi_asic - -CHECKPOINTS_DIR = "/etc/sonic/checkpoints" -CHECKPOINT_EXT = ".cp.json" - -def extract_scope(path): - if not path: - raise Exception("Wrong patch with empty path.") - - pointer = jsonpointer.JsonPointer(path) - parts = pointer.parts - - if not parts: - raise GenericConfigUpdaterError("Wrong patch with empty path.") - if parts[0].startswith("asic"): - if not parts[0][len("asic"):].isnumeric(): - raise GenericConfigUpdaterError(f"Error resolving path: '{path}' due to incorrect ASIC number.") - scope = parts[0] - remainder = "/" + "/".join(parts[1:]) - elif parts[0] == HOST_NAMESPACE: - scope = HOST_NAMESPACE - remainder = "/" + "/".join(parts[1:]) - else: - scope = "" - remainder = path - - return scope, remainder - - -def get_cmd_output(cmd): - proc = subprocess.Popen(cmd, text=True, stdout=subprocess.PIPE) - return proc.communicate()[0], proc.returncode - - -def get_config_json_by_namespace(namespace): - cmd = ['sonic-cfggen', '-d', '--print-data'] - if namespace is not None and namespace != multi_asic.DEFAULT_NAMESPACE: - cmd += ['-n', namespace] - - stdout, rc = get_cmd_output(cmd) - if rc: - raise GenericConfigUpdaterError("Failed to get cmd output '{}':rc {}".format(cmd, rc)) - - try: - config_json = json.loads(stdout) - except json.JSONDecodeError as e: - raise GenericConfigUpdaterError("Failed to get config by '{}' due to {}".format(cmd, e)) - - return config_json - - -class MultiASICConfigRollbacker: - def __init__(self, scopelist, checkpoints_dir=CHECKPOINTS_DIR): - self.logger = genericUpdaterLogging.get_logger(title="MultiASICConfigRollbacker", - print_all_to_console=True) - self.scopelist = scopelist - self.checkpoints_dir = checkpoints_dir - self.util = Util(checkpoints_dir=checkpoints_dir) - - def replace_all(self, target_config): - config_keys = set(target_config.keys()) - missing_scopes = set(self.scopelist) - config_keys - if missing_scopes: - raise GenericConfigUpdaterError(f"To be replace config is missing scope: {missing_scopes}") - - for scope in self.scopelist: - scope_config = target_config.pop(scope) - if scope.lower() == HOST_NAMESPACE: - scope = multi_asic.DEFAULT_NAMESPACE - ConfigReplacer(namespace=scope).replace(scope_config) - - def rollback_all(self, checkpoint_name): - self.logger.log_notice("Config rollbacking starting.") - self.logger.log_notice(f"Checkpoint name: {checkpoint_name}.") - self.logger.log_notice(f"Verifying '{checkpoint_name}' exists.") - if not self.util.check_checkpoint_exists(checkpoint_name): - raise ValueError(f"Checkpoint '{checkpoint_name}' does not exist") - self.logger.log_notice(f"Loading checkpoint '{checkpoint_name}' into memory.") - target_config = self.util.get_checkpoint_content(checkpoint_name) - self.logger.log_notice(f"Replacing config '{checkpoint_name}' using 'Config Replacer'.") - for scope in self.scopelist: - if scope.lower() == multi_asic.DEFAULT_NAMESPACE: - config = target_config.pop(HOST_NAMESPACE) - else: - config = target_config.pop(scope) - ConfigReplacer(namespace=scope).replace(config) - self.logger.log_notice("Config rollbacking completed.") - - def checkpoint_all(self, checkpoint_name): - all_configs = {} - self.logger.log_notice("Config checkpoint starting.") - self.logger.log_notice(f"Checkpoint name: {checkpoint_name}.") - for scope in self.scopelist: - self.logger.log_notice(f"Getting current {scope} config db.") - config = get_config_json_by_namespace(scope) - if scope.lower() == multi_asic.DEFAULT_NAMESPACE: - scope = HOST_NAMESPACE - all_configs[scope] = config - - self.logger.log_notice("Getting checkpoint full-path.") - path = self.util.get_checkpoint_full_path(checkpoint_name) - self.logger.log_notice("Ensuring checkpoint directory exist.") - self.util.ensure_checkpoints_dir_exists() - self.logger.log_notice(f"Saving config db content to {path}.") - - self.util.save_json_file(path, all_configs) - self.logger.log_notice("Config checkpoint completed.") - - def list_checkpoints(self): - self.logger.log_info("Listing checkpoints starting.") - - self.logger.log_info(f"Verifying checkpoints directory '{self.checkpoints_dir}' exists.") - if not self.util.checkpoints_dir_exist(): - self.logger.log_info("Checkpoints directory is empty, returning empty checkpoints list.") - return [] - - self.logger.log_info("Getting checkpoints in checkpoints directory.") - checkpoint_names = self.util.get_checkpoint_names() - - checkpoints_len = len(checkpoint_names) - self.logger.log_info( - f"Found {checkpoints_len} checkpoint{'s' if checkpoints_len != 1 else ''}" - f"{':' if checkpoints_len > 0 else '.'}" - ) - for checkpoint_name in checkpoint_names: - self.logger.log_info(f" * {checkpoint_name}") - - self.logger.log_info("Listing checkpoints completed.") - - return checkpoint_names - - def delete_checkpoint(self, checkpoint_name): - self.logger.log_notice("Deleting checkpoint starting.") - self.logger.log_notice(f"Checkpoint name: {checkpoint_name}.") - - self.logger.log_notice(f"Checking checkpoint: {checkpoint_name} exists.") - if not self.util.check_checkpoint_exists(checkpoint_name): - raise ValueError(f"Checkpoint '{checkpoint_name}' does not exist") - - self.logger.log_notice(f"Deleting checkpoint: {checkpoint_name}.") - self.util.delete_checkpoint(checkpoint_name) - - self.logger.log_notice(f"Deleting checkpoint: {checkpoint_name} completed.") - -class ConfigLock: - def acquire_lock(self): - # TODO: Implement ConfigLock - pass - - def release_lock(self): - # TODO: Implement ConfigLock - pass - - -class ConfigFormat(Enum): - CONFIGDB = 1 - SONICYANG = 2 - -class PatchApplier: - def __init__(self, - patchsorter=None, - changeapplier=None, - config_wrapper=None, - patch_wrapper=None, - namespace=multi_asic.DEFAULT_NAMESPACE): - self.namespace = namespace - self.logger = genericUpdaterLogging.get_logger(title="Patch Applier", print_all_to_console=True) - self.config_wrapper = config_wrapper if config_wrapper is not None else ConfigWrapper(namespace=self.namespace) - self.patch_wrapper = patch_wrapper if patch_wrapper is not None else PatchWrapper(namespace=self.namespace) - self.patchsorter = patchsorter if patchsorter is not None else StrictPatchSorter(self.config_wrapper, self.patch_wrapper) - self.changeapplier = changeapplier if changeapplier is not None else ChangeApplier(namespace=self.namespace) - - def apply(self, patch, sort=True): - scope = self.namespace if self.namespace else HOST_NAMESPACE - self.logger.log_notice(f"{scope}: Patch application starting.") - self.logger.log_notice(f"{scope}: Patch: {patch}") - - # Get old config - self.logger.log_notice(f"{scope} getting current config db.") - old_config = self.config_wrapper.get_config_db_as_json() - - # Generate target config - self.logger.log_notice(f"{scope}: simulating the target full config after applying the patch.") - target_config = self.patch_wrapper.simulate_patch(patch, old_config) - - # Validate all JsonPatch operations on specified fields - self.logger.log_notice(f"{scope}: validating all JsonPatch operations are permitted on the specified fields") - self.config_wrapper.validate_field_operation(old_config, target_config) - - # Validate target config does not have empty tables since they do not show up in ConfigDb - self.logger.log_notice(f"""{scope}: validating target config does not have empty tables, - since they do not show up in ConfigDb.""") - - empty_tables = self.config_wrapper.get_empty_tables(target_config) - if empty_tables: # if there are empty tables - empty_tables_txt = ", ".join(empty_tables) - raise EmptyTableError(f"{scope}: given patch is not valid because it will result in empty tables \ - which is not allowed in ConfigDb. \ - Table{'s' if len(empty_tables) != 1 else ''}: {empty_tables_txt}") - - # Generate list of changes to apply - if sort: - self.logger.log_notice(f"{scope}: sorting patch updates.") - changes = self.patchsorter.sort(patch) + import json + import subprocess + import jsonpointer + import os + from enum import Enum + from .gu_common import HOST_NAMESPACE, GenericConfigUpdaterError, EmptyTableError, ConfigWrapper, \ + DryRunConfigWrapper, PatchWrapper, genericUpdaterLogging + from .patch_sorter import StrictPatchSorter, NonStrictPatchSorter, ConfigSplitter, \ + TablesWithoutYangConfigSplitter, IgnorePathsFromYangConfigSplitter + from .change_applier import ChangeApplier, DryRunChangeApplier + from sonic_py_common import multi_asic + + CHECKPOINTS_DIR = "/etc/sonic/checkpoints" + CHECKPOINT_EXT = ".cp.json" + + def extract_scope(path): + if not path: + raise Exception("Wrong patch with empty path.") + + pointer = jsonpointer.JsonPointer(path) + parts = pointer.parts + + if not parts: + raise GenericConfigUpdaterError("Wrong patch with empty path.") + if parts[0].startswith("asic"): + if not parts[0][len("asic"):].isnumeric(): + raise GenericConfigUpdaterError(f"Error resolving path: '{path}' due to incorrect ASIC number.") + scope = parts[0] + remainder = "/" + "/".join(parts[1:]) + elif parts[0] == HOST_NAMESPACE: + scope = HOST_NAMESPACE + remainder = "/" + "/".join(parts[1:]) else: - self.logger.log_notice(f"{scope}: converting patch to JsonChange.") - changes = [JsonChange(jsonpatch.JsonPatch([element])) for element in patch] - - changes_len = len(changes) - self.logger.log_notice(f"The {scope} patch was converted into {changes_len} " \ - f"change{'s' if changes_len != 1 else ''}{':' if changes_len > 0 else '.'}") - - # Apply changes in order - self.logger.log_notice(f"{scope}: applying {changes_len} change{'s' if changes_len != 1 else ''} " \ - f"in order{':' if changes_len > 0 else '.'}") - for change in changes: - self.logger.log_notice(f" * {change}") - self.changeapplier.apply(change) - - # Validate config updated successfully - self.logger.log_notice(f"{scope}: verifying patch updates are reflected on ConfigDB.") - new_config = self.config_wrapper.get_config_db_as_json() - self.changeapplier.remove_backend_tables_from_config(target_config) - self.changeapplier.remove_backend_tables_from_config(new_config) - if not(self.patch_wrapper.verify_same_json(target_config, new_config)): - raise GenericConfigUpdaterError(f"{scope}: after applying patch to config, there are still some parts not updated") - - self.logger.log_notice(f"{scope} patch application completed.") - - -class ConfigReplacer: - def __init__(self, patch_applier=None, config_wrapper=None, patch_wrapper=None, namespace=multi_asic.DEFAULT_NAMESPACE): - self.namespace = namespace - self.logger = genericUpdaterLogging.get_logger(title="Config Replacer", print_all_to_console=True) - self.patch_applier = patch_applier if patch_applier is not None else PatchApplier(namespace=self.namespace) - self.config_wrapper = config_wrapper if config_wrapper is not None else ConfigWrapper(namespace=self.namespace) - self.patch_wrapper = patch_wrapper if patch_wrapper is not None else PatchWrapper(namespace=self.namespace) - - def replace(self, target_config): - self.logger.log_notice("Config replacement starting.") - self.logger.log_notice(f"Target config length: {len(json.dumps(target_config))}.") - - self.logger.log_notice("Getting current config db.") - old_config = self.config_wrapper.get_config_db_as_json() - - self.logger.log_notice("Generating patch between target config and current config db.") - patch = self.patch_wrapper.generate_patch(old_config, target_config) - self.logger.log_debug(f"Generated patch: {patch}.") # debug since the patch will printed again in 'patch_applier.apply' - - self.logger.log_notice("Applying patch using 'Patch Applier'.") - self.patch_applier.apply(patch) - - self.logger.log_notice("Verifying config replacement is reflected on ConfigDB.") - new_config = self.config_wrapper.get_config_db_as_json() - if not(self.patch_wrapper.verify_same_json(target_config, new_config)): - raise GenericConfigUpdaterError(f"After replacing config, there is still some parts not updated") - - self.logger.log_notice("Config replacement completed.") - - -class FileSystemConfigRollbacker: - def __init__(self, - checkpoints_dir=CHECKPOINTS_DIR, - config_replacer=None, - config_wrapper=None, - namespace=multi_asic.DEFAULT_NAMESPACE): - self.namespace = namespace - self.logger = genericUpdaterLogging.get_logger(title="Config Rollbacker", print_all_to_console=True) - self.util = Util(checkpoints_dir=checkpoints_dir) - self.checkpoints_dir = checkpoints_dir - self.config_replacer = config_replacer if config_replacer is not None else ConfigReplacer(namespace=self.namespace) - self.config_wrapper = config_wrapper if config_wrapper is not None else ConfigWrapper(namespace=self.namespace) - - def rollback(self, checkpoint_name): - self.logger.log_notice("Config rollbacking starting.") - self.logger.log_notice(f"Checkpoint name: {checkpoint_name}.") - - self.logger.log_notice(f"Verifying '{checkpoint_name}' exists.") - if not self.util.check_checkpoint_exists(checkpoint_name): - raise ValueError(f"Checkpoint '{checkpoint_name}' does not exist") - - self.logger.log_notice(f"Loading checkpoint into memory.") - target_config = self.util.get_checkpoint_content(checkpoint_name) - - self.logger.log_notice(f"Replacing config using 'Config Replacer'.") - self.config_replacer.replace(target_config) - - self.logger.log_notice("Config rollbacking completed.") - - def checkpoint(self, checkpoint_name): - self.logger.log_notice("Config checkpoint starting.") - self.logger.log_notice(f"Checkpoint name: {checkpoint_name}.") - - self.logger.log_notice("Getting current config db.") - json_content = self.config_wrapper.get_config_db_as_json() - - self.logger.log_notice("Getting checkpoint full-path.") - path = self.util.get_checkpoint_full_path(checkpoint_name) - - self.logger.log_notice("Ensuring checkpoint directory exist.") - self.util.ensure_checkpoints_dir_exists() + scope = "" + remainder = path - self.logger.log_notice(f"Saving config db content to {path}.") - self.util.save_json_file(path, json_content) + return scope, remainder - self.logger.log_notice("Config checkpoint completed.") - def list_checkpoints(self): - self.logger.log_info("Listing checkpoints starting.") + def get_cmd_output(cmd): + proc = subprocess.Popen(cmd, text=True, stdout=subprocess.PIPE) + return proc.communicate()[0], proc.returncode - self.logger.log_info(f"Verifying checkpoints directory '{self.checkpoints_dir}' exists.") - if not self.util.checkpoints_dir_exist(): - self.logger.log_info("Checkpoints directory is empty, returning empty checkpoints list.") - return [] - self.logger.log_info("Getting checkpoints in checkpoints directory.") - checkpoint_names = self.util.get_checkpoint_names() + def get_config_json_by_namespace(scope): + cmd = ['sonic-cfggen', '-d', '--print-data'] + if scope is not None and scope != multi_asic.DEFAULT_NAMESPACE: + cmd += ['-n', scope] - checkpoints_len = len(checkpoint_names) - self.logger.log_info(f"Found {checkpoints_len} checkpoint{'s' if checkpoints_len != 1 else ''}{':' if checkpoints_len > 0 else '.'}") - for checkpoint_name in checkpoint_names: - self.logger.log_info(f" * {checkpoint_name}") + stdout, rc = get_cmd_output(cmd) + if rc: + raise GenericConfigUpdaterError("Failed to get cmd output '{}':rc {}".format(cmd, rc)) - self.logger.log_info("Listing checkpoints completed.") + try: + config_json = json.loads(stdout) + except json.JSONDecodeError as e: + raise GenericConfigUpdaterError("Failed to get config by '{}' due to {}".format(cmd, e)) - return checkpoint_names + return config_json - def delete_checkpoint(self, checkpoint_name): - self.logger.log_notice("Deleting checkpoint starting.") - self.logger.log_notice(f"Checkpoint name: {checkpoint_name}.") - self.logger.log_notice(f"Checking checkpoint exists.") - if not self.util.check_checkpoint_exists(checkpoint_name): - raise ValueError(f"Checkpoint '{checkpoint_name}' does not exist") - - self.logger.log_notice(f"Deleting checkpoint.") - self.util.delete_checkpoint(checkpoint_name) - - self.logger.log_notice("Deleting checkpoint completed.") - - -class Util: - def __init__(self, checkpoints_dir=CHECKPOINTS_DIR): - self.checkpoints_dir = checkpoints_dir - - def ensure_checkpoints_dir_exists(self): - os.makedirs(self.checkpoints_dir, exist_ok=True) - - def save_json_file(self, path, json_content): - with open(path, "w") as fh: - fh.write(json.dumps(json_content)) - - def get_checkpoint_content(self, checkpoint_name): - path = self.get_checkpoint_full_path(checkpoint_name) - with open(path) as fh: - text = fh.read() - return json.loads(text) - - def get_checkpoint_full_path(self, name): - return os.path.join(self.checkpoints_dir, f"{name}{CHECKPOINT_EXT}") - - def get_checkpoint_names(self): - file_names = [] - for file_name in os.listdir(self.checkpoints_dir): - if file_name.endswith(CHECKPOINT_EXT): - # Remove extension from file name. - # Example assuming ext is '.cp.json', then 'checkpoint1.cp.json' becomes 'checkpoint1' - file_names.append(file_name[:-len(CHECKPOINT_EXT)]) - - return file_names - - def checkpoints_dir_exist(self): - return os.path.isdir(self.checkpoints_dir) + class ConfigLock: + def acquire_lock(self): + # TODO: Implement ConfigLock + pass - def check_checkpoint_exists(self, name): - path = self.get_checkpoint_full_path(name) - return os.path.isfile(path) + def release_lock(self): + # TODO: Implement ConfigLock + pass - def delete_checkpoint(self, name): - path = self.get_checkpoint_full_path(name) - return os.remove(path) + class ConfigFormat(Enum): + CONFIGDB = 1 + SONICYANG = 2 + + class PatchApplier: + def __init__(self, + patchsorter=None, + changeapplier=None, + config_wrapper=None, + patch_wrapper=None, + scope=multi_asic.DEFAULT_NAMESPACE): + self.scope = scope + self.logger = genericUpdaterLogging.get_logger(title="Patch Applier", print_all_to_console=True) + self.config_wrapper = config_wrapper if config_wrapper is not None else ConfigWrapper(scope=self.scope) + self.patch_wrapper = patch_wrapper if patch_wrapper is not None else PatchWrapper(scope=self.scope) + self.patchsorter = patchsorter if patchsorter is not None else StrictPatchSorter(self.config_wrapper, self.patch_wrapper) + self.changeapplier = changeapplier if changeapplier is not None else ChangeApplier(scope=self.scope) + + def apply(self, patch, sort=True): + scope = self.scope if self.scope else HOST_NAMESPACE + self.logger.log_notice(f"{scope}: Patch application starting.") + self.logger.log_notice(f"{scope}: Patch: {patch}") + + # Get old config + self.logger.log_notice(f"{scope} getting current config db.") + old_config = self.config_wrapper.get_config_db_as_json() + + # Generate target config + self.logger.log_notice(f"{scope}: simulating the target full config after applying the patch.") + target_config = self.patch_wrapper.simulate_patch(patch, old_config) + + # Validate all JsonPatch operations on specified fields + self.logger.log_notice(f"{scope}: validating all JsonPatch operations are permitted on the specified fields") + self.config_wrapper.validate_field_operation(old_config, target_config) + + # Validate target config does not have empty tables since they do not show up in ConfigDb + self.logger.log_notice(f"""{scope}: validating target config does not have empty tables, + since they do not show up in ConfigDb.""") + + empty_tables = self.config_wrapper.get_empty_tables(target_config) + if empty_tables: # if there are empty tables + empty_tables_txt = ", ".join(empty_tables) + raise EmptyTableError(f"{scope}: given patch is not valid because it will result in empty tables \ + which is not allowed in ConfigDb. \ + Table{'s' if len(empty_tables) != 1 else ''}: {empty_tables_txt}") + + # Generate list of changes to apply + if sort: + self.logger.log_notice(f"{scope}: sorting patch updates.") + changes = self.patchsorter.sort(patch) + else: + self.logger.log_notice(f"{scope}: converting patch to JsonChange.") + changes = [JsonChange(jsonpatch.JsonPatch([element])) for element in patch] -class Decorator(PatchApplier, ConfigReplacer, FileSystemConfigRollbacker): - def __init__(self, decorated_patch_applier=None, decorated_config_replacer=None, decorated_config_rollbacker=None, namespace=multi_asic.DEFAULT_NAMESPACE): - # initing base classes to make LGTM happy - PatchApplier.__init__(self, namespace=namespace) - ConfigReplacer.__init__(self, namespace=namespace) - FileSystemConfigRollbacker.__init__(self, namespace=namespace) + changes_len = len(changes) + self.logger.log_notice(f"The {scope} patch was converted into {changes_len} " \ + f"change{'s' if changes_len != 1 else ''}{':' if changes_len > 0 else '.'}") - self.decorated_patch_applier = decorated_patch_applier - self.decorated_config_replacer = decorated_config_replacer - self.decorated_config_rollbacker = decorated_config_rollbacker + # Apply changes in order + self.logger.log_notice(f"{scope}: applying {changes_len} change{'s' if changes_len != 1 else ''} " \ + f"in order{':' if changes_len > 0 else '.'}") + for change in changes: + self.logger.log_notice(f" * {change}") + self.changeapplier.apply(change) - def apply(self, patch): - self.decorated_patch_applier.apply(patch) + # Validate config updated successfully + self.logger.log_notice(f"{scope}: verifying patch updates are reflected on ConfigDB.") + new_config = self.config_wrapper.get_config_db_as_json() + self.changeapplier.remove_backend_tables_from_config(target_config) + self.changeapplier.remove_backend_tables_from_config(new_config) + if not(self.patch_wrapper.verify_same_json(target_config, new_config)): + raise GenericConfigUpdaterError(f"{scope}: after applying patch to config, there are still some parts not updated") - def replace(self, target_config): - self.decorated_config_replacer.replace(target_config) + self.logger.log_notice(f"{scope} patch application completed.") - def rollback(self, checkpoint_name): - self.decorated_config_rollbacker.rollback(checkpoint_name) - def checkpoint(self, checkpoint_name): - self.decorated_config_rollbacker.checkpoint(checkpoint_name) + class ConfigReplacer: + def __init__(self, patch_applier=None, config_wrapper=None, patch_wrapper=None, scope=multi_asic.DEFAULT_NAMESPACE): + self.scope = scope + self.logger = genericUpdaterLogging.get_logger(title="Config Replacer", print_all_to_console=True) + self.patch_applier = patch_applier if patch_applier is not None else PatchApplier(scope=self.scope) + self.config_wrapper = config_wrapper if config_wrapper is not None else ConfigWrapper(scope=self.scope) + self.patch_wrapper = patch_wrapper if patch_wrapper is not None else PatchWrapper(scope=self.scope) - def list_checkpoints(self): - return self.decorated_config_rollbacker.list_checkpoints() + def replace(self, target_config): + self.logger.log_notice("Config replacement starting.") + self.logger.log_notice(f"Target config length: {len(json.dumps(target_config))}.") - def delete_checkpoint(self, checkpoint_name): - self.decorated_config_rollbacker.delete_checkpoint(checkpoint_name) + self.logger.log_notice("Getting current config db.") + old_config = self.config_wrapper.get_config_db_as_json() + self.logger.log_notice("Generating patch between target config and current config db.") + patch = self.patch_wrapper.generate_patch(old_config, target_config) + self.logger.log_debug(f"Generated patch: {patch}.") # debug since the patch will printed again in 'patch_applier.apply' -class SonicYangDecorator(Decorator): - def __init__(self, patch_wrapper, config_wrapper, decorated_patch_applier=None, decorated_config_replacer=None, namespace=multi_asic.DEFAULT_NAMESPACE): - Decorator.__init__(self, decorated_patch_applier, decorated_config_replacer, namespace=namespace) + self.logger.log_notice("Applying patch using 'Patch Applier'.") + self.patch_applier.apply(patch) - self.namespace = namespace - self.patch_wrapper = patch_wrapper - self.config_wrapper = config_wrapper + self.logger.log_notice("Verifying config replacement is reflected on ConfigDB.") + new_config = self.config_wrapper.get_config_db_as_json() + if not(self.patch_wrapper.verify_same_json(target_config, new_config)): + raise GenericConfigUpdaterError(f"After replacing config, there is still some parts not updated") - def apply(self, patch): - config_db_patch = self.patch_wrapper.convert_sonic_yang_patch_to_config_db_patch(patch) - Decorator.apply(self, config_db_patch) + self.logger.log_notice("Config replacement completed.") - def replace(self, target_config): - config_db_target_config = self.config_wrapper.convert_sonic_yang_to_config_db(target_config) - Decorator.replace(self, config_db_target_config) + class FileSystemConfigRollbacker: + def __init__(self, + checkpoints_dir=CHECKPOINTS_DIR, + config_replacer=None, + config_wrapper=None, + scope=multi_asic.DEFAULT_NAMESPACE): + self.scope = scope + self.logger = genericUpdaterLogging.get_logger(title="Config Rollbacker", print_all_to_console=True) + self.util = Util(checkpoints_dir=checkpoints_dir) + self.checkpoints_dir = checkpoints_dir + self.config_replacer = config_replacer if config_replacer is not None else ConfigReplacer(scope=self.scope) + self.config_wrapper = config_wrapper if config_wrapper is not None else ConfigWrapper(scope=self.scope) -class ConfigLockDecorator(Decorator): - def __init__(self, - decorated_patch_applier=None, - decorated_config_replacer=None, - decorated_config_rollbacker=None, - config_lock=ConfigLock(), - namespace=multi_asic.DEFAULT_NAMESPACE): - Decorator.__init__(self, decorated_patch_applier, decorated_config_replacer, decorated_config_rollbacker, namespace=namespace) + def rollback(self, checkpoint_name): + self.logger.log_notice("Config rollbacking starting.") + self.logger.log_notice(f"Checkpoint name: {checkpoint_name}.") + + self.logger.log_notice(f"Verifying '{checkpoint_name}' exists.") + if not self.util.check_checkpoint_exists(checkpoint_name): + raise ValueError(f"Checkpoint '{checkpoint_name}' does not exist") + + self.logger.log_notice(f"Loading checkpoint into memory.") + target_config = self.util.get_checkpoint_content(checkpoint_name) + + self.logger.log_notice(f"Replacing config using 'Config Replacer'.") + self.config_replacer.replace(target_config) + + self.logger.log_notice("Config rollbacking completed.") + + def checkpoint(self, checkpoint_name): + self.logger.log_notice("Config checkpoint starting.") + self.logger.log_notice(f"Checkpoint name: {checkpoint_name}.") + + self.logger.log_notice("Getting current config db.") + json_content = self.config_wrapper.get_config_db_as_json() + + self.logger.log_notice("Getting checkpoint full-path.") + path = self.util.get_checkpoint_full_path(checkpoint_name) + + self.logger.log_notice("Ensuring checkpoint directory exist.") + self.util.ensure_checkpoints_dir_exists() + + self.logger.log_notice(f"Saving config db content to {path}.") + self.util.save_json_file(path, json_content) + + self.logger.log_notice("Config checkpoint completed.") + + def list_checkpoints(self): + self.logger.log_info("Listing checkpoints starting.") + + self.logger.log_info(f"Verifying checkpoints directory '{self.checkpoints_dir}' exists.") + if not self.util.checkpoints_dir_exist(): + self.logger.log_info("Checkpoints directory is empty, returning empty checkpoints list.") + return [] + + self.logger.log_info("Getting checkpoints in checkpoints directory.") + checkpoint_names = self.util.get_checkpoint_names() + + checkpoints_len = len(checkpoint_names) + self.logger.log_info(f"Found {checkpoints_len} checkpoint{'s' if checkpoints_len != 1 else ''}{':' if checkpoints_len > 0 else '.'}") + for checkpoint_name in checkpoint_names: + self.logger.log_info(f" * {checkpoint_name}") + + self.logger.log_info("Listing checkpoints completed.") + + return checkpoint_names + + def delete_checkpoint(self, checkpoint_name): + self.logger.log_notice("Deleting checkpoint starting.") + self.logger.log_notice(f"Checkpoint name: {checkpoint_name}.") + + self.logger.log_notice(f"Checking checkpoint exists.") + if not self.util.check_checkpoint_exists(checkpoint_name): + raise ValueError(f"Checkpoint '{checkpoint_name}' does not exist") + + self.logger.log_notice(f"Deleting checkpoint.") + self.util.delete_checkpoint(checkpoint_name) + + self.logger.log_notice("Deleting checkpoint completed.") + + + class MultiASICConfigReplacer: + def __init__(self, checkpoints_dir=CHECKPOINTS_DIR): + self.logger = genericUpdaterLogging.get_logger(title="MultiASICConfigReplacer", + print_all_to_console=True) + self.scopelist = [HOST_NAMESPACE].extend(multi_asic.get_namespace_list()) + self.checkpoints_dir = checkpoints_dir + self.util = Util(checkpoints_dir=checkpoints_dir) + + def replace(self, target_config): + config_keys = set(target_config.keys()) + missing_scopes = set(self.scopelist) - config_keys + if missing_scopes: + raise GenericConfigUpdaterError(f"To be replace config is missing scope: {missing_scopes}") + + for scope in self.scopelist: + scope_config = target_config.pop(scope) + if scope.lower() == HOST_NAMESPACE: + scope = multi_asic.DEFAULT_NAMESPACE + ConfigReplacer(namespace=scope).replace(scope_config) + + + class MultiASICConfigRollbacker(FileSystemConfigRollbacker): + def __init__(self, checkpoints_dir=CHECKPOINTS_DIR): + self.logger = genericUpdaterLogging.get_logger(title="MultiASICConfigRollbacker", + print_all_to_console=True) + self.scopelist = [HOST_NAMESPACE].extend(multi_asic.get_namespace_list()) + self.checkpoints_dir = checkpoints_dir + self.util = Util(checkpoints_dir=checkpoints_dir) + + def rollback(self, checkpoint_name): + self.logger.log_notice("Config rollbacking starting.") + self.logger.log_notice(f"Checkpoint name: {checkpoint_name}.") + self.logger.log_notice(f"Verifying '{checkpoint_name}' exists.") + if not self.util.check_checkpoint_exists(checkpoint_name): + raise ValueError(f"Checkpoint '{checkpoint_name}' does not exist") + self.logger.log_notice(f"Loading checkpoint '{checkpoint_name}' into memory.") + target_config = self.util.get_checkpoint_content(checkpoint_name) + self.logger.log_notice(f"Replacing config '{checkpoint_name}' using 'Config Replacer'.") + for scope in self.scopelist: + if scope.lower() == multi_asic.DEFAULT_NAMESPACE: + config = target_config.pop(HOST_NAMESPACE) + else: + config = target_config.pop(scope) + ConfigReplacer(namespace=scope).replace(config) + self.logger.log_notice("Config rollbacking completed.") - self.config_lock = config_lock + def checkpoint(self, checkpoint_name): + all_configs = {} + self.logger.log_notice("Config checkpoint starting.") + self.logger.log_notice(f"Checkpoint name: {checkpoint_name}.") + for scope in self.scopelist: + self.logger.log_notice(f"Getting current {scope} config db.") + config = get_config_json_by_namespace(scope) + if scope.lower() == multi_asic.DEFAULT_NAMESPACE: + scope = HOST_NAMESPACE + all_configs[scope] = config - def apply(self, patch, sort=True): - self.execute_write_action(Decorator.apply, self, patch) + self.logger.log_notice("Getting checkpoint full-path.") + path = self.util.get_checkpoint_full_path(checkpoint_name) + self.logger.log_notice("Ensuring checkpoint directory exist.") + self.util.ensure_checkpoints_dir_exists() + self.logger.log_notice(f"Saving config db content to {path}.") + + self.util.save_json_file(path, all_configs) + self.logger.log_notice("Config checkpoint completed.") + + + class Util: + def __init__(self, checkpoints_dir=CHECKPOINTS_DIR): + self.checkpoints_dir = checkpoints_dir + + def ensure_checkpoints_dir_exists(self): + os.makedirs(self.checkpoints_dir, exist_ok=True) + + def save_json_file(self, path, json_content): + with open(path, "w") as fh: + fh.write(json.dumps(json_content)) + + def get_checkpoint_content(self, checkpoint_name): + path = self.get_checkpoint_full_path(checkpoint_name) + with open(path) as fh: + text = fh.read() + return json.loads(text) + + def get_checkpoint_full_path(self, name): + return os.path.join(self.checkpoints_dir, f"{name}{CHECKPOINT_EXT}") + + def get_checkpoint_names(self): + file_names = [] + for file_name in os.listdir(self.checkpoints_dir): + if file_name.endswith(CHECKPOINT_EXT): + # Remove extension from file name. + # Example assuming ext is '.cp.json', then 'checkpoint1.cp.json' becomes 'checkpoint1' + file_names.append(file_name[:-len(CHECKPOINT_EXT)]) + + return file_names + + def checkpoints_dir_exist(self): + return os.path.isdir(self.checkpoints_dir) + + def check_checkpoint_exists(self, name): + path = self.get_checkpoint_full_path(name) + return os.path.isfile(path) + + def delete_checkpoint(self, name): + path = self.get_checkpoint_full_path(name) + return os.remove(path) + + + class Decorator(PatchApplier, ConfigReplacer, FileSystemConfigRollbacker): + def __init__(self, decorated_patch_applier=None, decorated_config_replacer=None, decorated_config_rollbacker=None, scope=multi_asic.DEFAULT_NAMESPACE): + # initing base classes to make LGTM happy + PatchApplier.__init__(self, scope=scope) + ConfigReplacer.__init__(self, scope=scope) + FileSystemConfigRollbacker.__init__(self, scope=scope) + + self.decorated_patch_applier = decorated_patch_applier + self.decorated_config_replacer = decorated_config_replacer + self.decorated_config_rollbacker = decorated_config_rollbacker + + def apply(self, patch): + self.decorated_patch_applier.apply(patch) + + def replace(self, target_config): + self.decorated_config_replacer.replace(target_config) + + def rollback(self, checkpoint_name): + self.decorated_config_rollbacker.rollback(checkpoint_name) + + def checkpoint(self, checkpoint_name): + self.decorated_config_rollbacker.checkpoint(checkpoint_name) + + def list_checkpoints(self): + return self.decorated_config_rollbacker.list_checkpoints() + + def delete_checkpoint(self, checkpoint_name): + self.decorated_config_rollbacker.delete_checkpoint(checkpoint_name) + + + class SonicYangDecorator(Decorator): + def __init__(self, patch_wrapper, config_wrapper, decorated_patch_applier=None, decorated_config_replacer=None, scope=multi_asic.DEFAULT_NAMESPACE): + Decorator.__init__(self, decorated_patch_applier, decorated_config_replacer, scope=scope) + + self.scope = scope + self.patch_wrapper = patch_wrapper + self.config_wrapper = config_wrapper + + def apply(self, patch): + config_db_patch = self.patch_wrapper.convert_sonic_yang_patch_to_config_db_patch(patch) + Decorator.apply(self, config_db_patch) - def replace(self, target_config): - self.execute_write_action(Decorator.replace, self, target_config) + def replace(self, target_config): + config_db_target_config = self.config_wrapper.convert_sonic_yang_to_config_db(target_config) + Decorator.replace(self, config_db_target_config) - def rollback(self, checkpoint_name): - self.execute_write_action(Decorator.rollback, self, checkpoint_name) - def checkpoint(self, checkpoint_name): - self.execute_write_action(Decorator.checkpoint, self, checkpoint_name) + class ConfigLockDecorator(Decorator): + def __init__(self, + decorated_patch_applier=None, + decorated_config_replacer=None, + decorated_config_rollbacker=None, + config_lock=ConfigLock(), + scope=multi_asic.DEFAULT_NAMESPACE): + Decorator.__init__(self, decorated_patch_applier, decorated_config_replacer, decorated_config_rollbacker, scope=scope) - def execute_write_action(self, action, *args): - self.config_lock.acquire_lock() - action(*args) - self.config_lock.release_lock() + self.config_lock = config_lock + def apply(self, patch, sort=True): + self.execute_write_action(Decorator.apply, self, patch) -class GenericUpdateFactory: - def __init__(self, namespace=multi_asic.DEFAULT_NAMESPACE): - self.namespace = namespace + def replace(self, target_config): + self.execute_write_action(Decorator.replace, self, target_config) - def create_patch_applier(self, config_format, verbose, dry_run, ignore_non_yang_tables, ignore_paths): - self.init_verbose_logging(verbose) - config_wrapper = self.get_config_wrapper(dry_run) - change_applier = self.get_change_applier(dry_run, config_wrapper) - patch_wrapper = PatchWrapper(config_wrapper, namespace=self.namespace) - patch_sorter = self.get_patch_sorter(ignore_non_yang_tables, ignore_paths, config_wrapper, patch_wrapper) - patch_applier = PatchApplier(config_wrapper=config_wrapper, - patchsorter=patch_sorter, - patch_wrapper=patch_wrapper, - changeapplier=change_applier, - namespace=self.namespace) + def rollback(self, checkpoint_name): + self.execute_write_action(Decorator.rollback, self, checkpoint_name) - if config_format == ConfigFormat.CONFIGDB: - pass - elif config_format == ConfigFormat.SONICYANG: - patch_applier = SonicYangDecorator(decorated_patch_applier=patch_applier, - patch_wrapper=patch_wrapper, - config_wrapper=config_wrapper, - namespace=self.namespace) - else: - raise ValueError(f"config-format '{config_format}' is not supported") + def checkpoint(self, checkpoint_name): + self.execute_write_action(Decorator.checkpoint, self, checkpoint_name) - if not dry_run: - patch_applier = ConfigLockDecorator(decorated_patch_applier=patch_applier, namespace=self.namespace) + def execute_write_action(self, action, *args): + self.config_lock.acquire_lock() + action(*args) + self.config_lock.release_lock() - return patch_applier - def create_config_replacer(self, config_format, verbose, dry_run, ignore_non_yang_tables, ignore_paths): - self.init_verbose_logging(verbose) + class GenericUpdateFactory: + def __init__(self, scope=multi_asic.DEFAULT_NAMESPACE): + self.scope = scope - config_wrapper = self.get_config_wrapper(dry_run) - change_applier = self.get_change_applier(dry_run, config_wrapper) - patch_wrapper = PatchWrapper(config_wrapper, namespace=self.namespace) - patch_sorter = self.get_patch_sorter(ignore_non_yang_tables, ignore_paths, config_wrapper, patch_wrapper) - patch_applier = PatchApplier(config_wrapper=config_wrapper, - patchsorter=patch_sorter, - patch_wrapper=patch_wrapper, - changeapplier=change_applier, - namespace=self.namespace) + def create_patch_applier(self, config_format, verbose, dry_run, ignore_non_yang_tables, ignore_paths): + self.init_verbose_logging(verbose) + config_wrapper = self.get_config_wrapper(dry_run) + change_applier = self.get_change_applier(dry_run, config_wrapper) + patch_wrapper = PatchWrapper(config_wrapper, scope=self.scope) + patch_sorter = self.get_patch_sorter(ignore_non_yang_tables, ignore_paths, config_wrapper, patch_wrapper) + patch_applier = PatchApplier(config_wrapper=config_wrapper, + patchsorter=patch_sorter, + patch_wrapper=patch_wrapper, + changeapplier=change_applier, + scope=self.scope) - config_replacer = ConfigReplacer(patch_applier=patch_applier, config_wrapper=config_wrapper, namespace=self.namespace) - if config_format == ConfigFormat.CONFIGDB: - pass - elif config_format == ConfigFormat.SONICYANG: - config_replacer = SonicYangDecorator(decorated_config_replacer=config_replacer, - patch_wrapper=patch_wrapper, - config_wrapper=config_wrapper, - namespace=self.namespace) - else: - raise ValueError(f"config-format '{config_format}' is not supported") + if config_format == ConfigFormat.CONFIGDB: + pass + elif config_format == ConfigFormat.SONICYANG: + patch_applier = SonicYangDecorator(decorated_patch_applier=patch_applier, + patch_wrapper=patch_wrapper, + config_wrapper=config_wrapper, + scope=self.scope) + else: + raise ValueError(f"config-format '{config_format}' is not supported") + + if not dry_run: + patch_applier = ConfigLockDecorator(decorated_patch_applier=patch_applier, scope=self.scope) + + return patch_applier + + def create_config_replacer(self, config_format, verbose, dry_run, ignore_non_yang_tables, ignore_paths): + self.init_verbose_logging(verbose) + + config_wrapper = self.get_config_wrapper(dry_run) + change_applier = self.get_change_applier(dry_run, config_wrapper) + patch_wrapper = PatchWrapper(config_wrapper, scope=self.scope) + patch_sorter = self.get_patch_sorter(ignore_non_yang_tables, ignore_paths, config_wrapper, patch_wrapper) + patch_applier = PatchApplier(config_wrapper=config_wrapper, + patchsorter=patch_sorter, + patch_wrapper=patch_wrapper, + changeapplier=change_applier, + scope=self.scope) + + config_replacer = MultiASICConfigReplacer() if multi_asic.is_multi_asic() else ConfigReplacer( + patch_applier=patch_applier, config_wrapper=config_wrapper, scope=self.scope) + + if config_format == ConfigFormat.CONFIGDB: + pass + elif config_format == ConfigFormat.SONICYANG: + config_replacer = SonicYangDecorator(decorated_config_replacer=config_replacer, + patch_wrapper=patch_wrapper, + config_wrapper=config_wrapper, + scope=self.scope) + else: + raise ValueError(f"config-format '{config_format}' is not supported") - if not dry_run: - config_replacer = ConfigLockDecorator(decorated_config_replacer=config_replacer, namespace=self.namespace) + if not dry_run: + config_replacer = ConfigLockDecorator(decorated_config_replacer=config_replacer, scope=self.scope) - return config_replacer + return config_replacer - def create_config_rollbacker(self, verbose, dry_run=False, ignore_non_yang_tables=False, ignore_paths=[]): - self.init_verbose_logging(verbose) + def create_config_rollbacker(self, verbose, dry_run=False, ignore_non_yang_tables=False, ignore_paths=[]): + self.init_verbose_logging(verbose) - config_wrapper = self.get_config_wrapper(dry_run) - change_applier = self.get_change_applier(dry_run, config_wrapper) - patch_wrapper = PatchWrapper(config_wrapper, namespace=self.namespace) - patch_sorter = self.get_patch_sorter(ignore_non_yang_tables, ignore_paths, config_wrapper, patch_wrapper) - patch_applier = PatchApplier(config_wrapper=config_wrapper, - patchsorter=patch_sorter, - patch_wrapper=patch_wrapper, - changeapplier=change_applier, - namespace=self.namespace) + config_wrapper = self.get_config_wrapper(dry_run) + change_applier = self.get_change_applier(dry_run, config_wrapper) + patch_wrapper = PatchWrapper(config_wrapper, scope=self.scope) + patch_sorter = self.get_patch_sorter(ignore_non_yang_tables, ignore_paths, config_wrapper, patch_wrapper) + patch_applier = PatchApplier(config_wrapper=config_wrapper, + patchsorter=patch_sorter, + patch_wrapper=patch_wrapper, + changeapplier=change_applier, + scope=self.scope) - config_replacer = ConfigReplacer(config_wrapper=config_wrapper, patch_applier=patch_applier, namespace=self.namespace) - config_rollbacker = FileSystemConfigRollbacker(config_wrapper=config_wrapper, config_replacer=config_replacer, namespace=self.namespace) + config_replacer = MultiASICConfigReplacer() if multi_asic.is_multi_asic() else ConfigReplacer( + config_wrapper=config_wrapper, patch_applier=patch_applier, scope=self.scope) + config_rollbacker = MultiASICConfigRollbacker() if multi_asic.is_multi_asic() else FileSystemConfigRollbacker( + config_wrapper=config_wrapper, config_replacer=config_replacer, scope=self.scope) - if not dry_run: - config_rollbacker = ConfigLockDecorator(decorated_config_rollbacker=config_rollbacker, namespace=self.namespace) + if not dry_run: + config_rollbacker = ConfigLockDecorator(decorated_config_rollbacker=config_rollbacker, scope=self.scope) - return config_rollbacker + return config_rollbacker - def init_verbose_logging(self, verbose): - genericUpdaterLogging.set_verbose(verbose) + def init_verbose_logging(self, verbose): + genericUpdaterLogging.set_verbose(verbose) - def get_config_wrapper(self, dry_run): - if dry_run: - return DryRunConfigWrapper(namespace=self.namespace) - else: - return ConfigWrapper(namespace=self.namespace) + def get_config_wrapper(self, dry_run): + if dry_run: + return DryRunConfigWrapper(scope=self.scope) + else: + return ConfigWrapper(scope=self.scope) - def get_change_applier(self, dry_run, config_wrapper): - if dry_run: - return DryRunChangeApplier(config_wrapper) - else: - return ChangeApplier(namespace=self.namespace) + def get_change_applier(self, dry_run, config_wrapper): + if dry_run: + return DryRunChangeApplier(config_wrapper) + else: + return ChangeApplier(scope=self.scope) - def get_patch_sorter(self, ignore_non_yang_tables, ignore_paths, config_wrapper, patch_wrapper): - if not ignore_non_yang_tables and not ignore_paths: - return StrictPatchSorter(config_wrapper, patch_wrapper) + def get_patch_sorter(self, ignore_non_yang_tables, ignore_paths, config_wrapper, patch_wrapper): + if not ignore_non_yang_tables and not ignore_paths: + return StrictPatchSorter(config_wrapper, patch_wrapper) - inner_config_splitters = [] - if ignore_non_yang_tables: - inner_config_splitters.append(TablesWithoutYangConfigSplitter(config_wrapper)) + inner_config_splitters = [] + if ignore_non_yang_tables: + inner_config_splitters.append(TablesWithoutYangConfigSplitter(config_wrapper)) - if ignore_paths: - inner_config_splitters.append(IgnorePathsFromYangConfigSplitter(ignore_paths, config_wrapper)) + if ignore_paths: + inner_config_splitters.append(IgnorePathsFromYangConfigSplitter(ignore_paths, config_wrapper)) - config_splitter = ConfigSplitter(config_wrapper, inner_config_splitters) + config_splitter = ConfigSplitter(config_wrapper, inner_config_splitters) - return NonStrictPatchSorter(config_wrapper, patch_wrapper, config_splitter) + return NonStrictPatchSorter(config_wrapper, patch_wrapper, config_splitter) -class GenericUpdater: - def __init__(self, generic_update_factory=None, namespace=multi_asic.DEFAULT_NAMESPACE): - self.generic_update_factory = \ - generic_update_factory if generic_update_factory is not None else GenericUpdateFactory(namespace=namespace) + class GenericUpdater: + def __init__(self, generic_update_factory=None, scope=multi_asic.DEFAULT_NAMESPACE): + self.generic_update_factory = \ + generic_update_factory if generic_update_factory is not None else GenericUpdateFactory(scope=scope) - def apply_patch(self, patch, config_format, verbose, dry_run, ignore_non_yang_tables, ignore_paths, sort=True): - patch_applier = self.generic_update_factory.create_patch_applier(config_format, verbose, dry_run, ignore_non_yang_tables, ignore_paths) - patch_applier.apply(patch, sort) + def apply_patch(self, patch, config_format, verbose, dry_run, ignore_non_yang_tables, ignore_paths, sort=True): + patch_applier = self.generic_update_factory.create_patch_applier(config_format, verbose, dry_run, ignore_non_yang_tables, ignore_paths) + patch_applier.apply(patch, sort) - def replace(self, target_config, config_format, verbose, dry_run, ignore_non_yang_tables, ignore_paths): - config_replacer = self.generic_update_factory.create_config_replacer(config_format, verbose, dry_run, ignore_non_yang_tables, ignore_paths) - config_replacer.replace(target_config) + def replace(self, target_config, config_format, verbose, dry_run, ignore_non_yang_tables, ignore_paths): + config_replacer = self.generic_update_factory.create_config_replacer(config_format, verbose, dry_run, ignore_non_yang_tables, ignore_paths) + config_replacer.replace(target_config) - def rollback(self, checkpoint_name, verbose, dry_run, ignore_non_yang_tables, ignore_paths): - config_rollbacker = self.generic_update_factory.create_config_rollbacker(verbose, dry_run, ignore_non_yang_tables, ignore_paths) - config_rollbacker.rollback(checkpoint_name) + def rollback(self, checkpoint_name, verbose, dry_run, ignore_non_yang_tables, ignore_paths): + config_rollbacker = self.generic_update_factory.create_config_rollbacker(verbose, dry_run, ignore_non_yang_tables, ignore_paths) + config_rollbacker.rollback(checkpoint_name) - def checkpoint(self, checkpoint_name, verbose): - config_rollbacker = self.generic_update_factory.create_config_rollbacker(verbose) - config_rollbacker.checkpoint(checkpoint_name) + def checkpoint(self, checkpoint_name, verbose): + config_rollbacker = self.generic_update_factory.create_config_rollbacker(verbose) + config_rollbacker.checkpoint(checkpoint_name) - def delete_checkpoint(self, checkpoint_name, verbose): - config_rollbacker = self.generic_update_factory.create_config_rollbacker(verbose) - config_rollbacker.delete_checkpoint(checkpoint_name) + def delete_checkpoint(self, checkpoint_name, verbose): + config_rollbacker = self.generic_update_factory.create_config_rollbacker(verbose) + config_rollbacker.delete_checkpoint(checkpoint_name) - def list_checkpoints(self, verbose): - config_rollbacker = self.generic_update_factory.create_config_rollbacker(verbose) - return config_rollbacker.list_checkpoints() + def list_checkpoints(self, verbose): + config_rollbacker = self.generic_update_factory.create_config_rollbacker(verbose) + return config_rollbacker.list_checkpoints() diff --git a/generic_config_updater/gu_common.py b/generic_config_updater/gu_common.py index c15334222a..54c54c9539 100644 --- a/generic_config_updater/gu_common.py +++ b/generic_config_updater/gu_common.py @@ -53,8 +53,8 @@ def __eq__(self, other): return False class ConfigWrapper: - def __init__(self, yang_dir=YANG_DIR, namespace=multi_asic.DEFAULT_NAMESPACE): - self.namespace = namespace + def __init__(self, yang_dir=YANG_DIR, scope=multi_asic.DEFAULT_NAMESPACE): + self.scope = scope self.yang_dir = YANG_DIR self.sonic_yang_with_loaded_models = None @@ -65,8 +65,8 @@ def get_config_db_as_json(self): return config_db_json def _get_config_db_as_text(self): - if self.namespace is not None and self.namespace != multi_asic.DEFAULT_NAMESPACE: - cmd = ['sonic-cfggen', '-d', '--print-data', '-n', self.namespace] + if self.scope is not None and self.scope != multi_asic.DEFAULT_NAMESPACE: + cmd = ['sonic-cfggen', '-d', '--print-data', '-n', self.scope] else: cmd = ['sonic-cfggen', '-d', '--print-data'] @@ -74,7 +74,7 @@ def _get_config_db_as_text(self): text, err = result.communicate() return_code = result.returncode if return_code: # non-zero means failure - raise GenericConfigUpdaterError(f"Failed to get running config for namespace: {self.namespace}, Return code: {return_code}, Error: {err}") + raise GenericConfigUpdaterError(f"Failed to get running config for scope: {self.scope}, Return code: {return_code}, Error: {err}") return text def get_sonic_yang_as_json(self): @@ -301,8 +301,8 @@ def create_sonic_yang_with_loaded_models(self): class DryRunConfigWrapper(ConfigWrapper): # This class will simulate all read/write operations to ConfigDB on a virtual storage unit. - def __init__(self, initial_imitated_config_db = None, namespace=multi_asic.DEFAULT_NAMESPACE): - super().__init__(namespace=namespace) + def __init__(self, initial_imitated_config_db = None, scope=multi_asic.DEFAULT_NAMESPACE): + super().__init__(scope=scope) self.logger = genericUpdaterLogging.get_logger(title="** DryRun", print_all_to_console=True) self.imitated_config_db = copy.deepcopy(initial_imitated_config_db) @@ -322,9 +322,9 @@ def _init_imitated_config_db_if_none(self): class PatchWrapper: - def __init__(self, config_wrapper=None, namespace=multi_asic.DEFAULT_NAMESPACE): - self.namespace = namespace - self.config_wrapper = config_wrapper if config_wrapper is not None else ConfigWrapper(self.namespace) + def __init__(self, config_wrapper=None, scope=multi_asic.DEFAULT_NAMESPACE): + self.scope = scope + self.config_wrapper = config_wrapper if config_wrapper is not None else ConfigWrapper(self.scope) self.path_addressing = PathAddressing(self.config_wrapper) def validate_config_db_patch_has_yang_models(self, patch): diff --git a/tests/config_test.py b/tests/config_test.py index a9cc33a601..cfdbdac7d1 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -2885,7 +2885,7 @@ def test_repalce_multiasic(self): mock_replace_content = copy.deepcopy(self.all_config) with patch('builtins.open', mock_open(read_data=json.dumps(mock_replace_content)), create=True) as mocked_open: # Mock GenericUpdater to avoid actual patch application - with patch('config.main.MultiASICConfigRollbacker') as mock_generic_updater: + with patch('config.main.GenericUpdater') as mock_generic_updater: mock_generic_updater.return_value.replace_all = MagicMock() print("Multi ASIC: {}".format(multi_asic.is_multi_asic())) @@ -2979,25 +2979,6 @@ def test_rollback_multiasic(self, mock_get_checkpoint_content): self.assertEqual(result.exit_code, 0, "Command should succeed") self.assertIn("Config rolled back successfully.", result.output) - @patch('generic_config_updater.generic_updater.Util.delete_checkpoint', MagicMock()) - @patch('generic_config_updater.generic_updater.Util.check_checkpoint_exists', mock.Mock(return_value=True)) - def test_delete_checkpoint_multiasic(self): - checkpointname = "checkpointname" - # Mock GenericUpdater to avoid actual patch application - with patch('config.main.GenericUpdater') as mock_generic_updater: - mock_generic_updater.return_value.deletecheckpoint = MagicMock() - - print("Multi ASIC: {}".format(multi_asic.is_multi_asic())) - # Invocation of the command with the CliRunner - result = self.runner.invoke(config.config.commands["delete-checkpoint"], - [checkpointname], - catch_exceptions=True) - - print("Exit Code: {}, output: {}".format(result.exit_code, result.output)) - # Assertions and verifications - self.assertEqual(result.exit_code, 0, "Command should succeed") - self.assertIn("Checkpoint deleted successfully.", result.output) - @classmethod def teardown_class(cls): print("TEARDOWN") From e66d9af1762d1725d521901492de5a6957cb62e8 Mon Sep 17 00:00:00 2001 From: Xincun Li Date: Wed, 29 May 2024 11:57:14 -0700 Subject: [PATCH 26/44] fix generic_updater format --- generic_config_updater/generic_updater.py | 1032 ++++++++++----------- 1 file changed, 513 insertions(+), 519 deletions(-) diff --git a/generic_config_updater/generic_updater.py b/generic_config_updater/generic_updater.py index 7ca9acdc99..c684a63b1f 100644 --- a/generic_config_updater/generic_updater.py +++ b/generic_config_updater/generic_updater.py @@ -1,587 +1,581 @@ - import json - import subprocess - import jsonpointer - import os - from enum import Enum - from .gu_common import HOST_NAMESPACE, GenericConfigUpdaterError, EmptyTableError, ConfigWrapper, \ - DryRunConfigWrapper, PatchWrapper, genericUpdaterLogging - from .patch_sorter import StrictPatchSorter, NonStrictPatchSorter, ConfigSplitter, \ - TablesWithoutYangConfigSplitter, IgnorePathsFromYangConfigSplitter - from .change_applier import ChangeApplier, DryRunChangeApplier - from sonic_py_common import multi_asic - - CHECKPOINTS_DIR = "/etc/sonic/checkpoints" - CHECKPOINT_EXT = ".cp.json" - - def extract_scope(path): - if not path: - raise Exception("Wrong patch with empty path.") - - pointer = jsonpointer.JsonPointer(path) - parts = pointer.parts - - if not parts: - raise GenericConfigUpdaterError("Wrong patch with empty path.") - if parts[0].startswith("asic"): - if not parts[0][len("asic"):].isnumeric(): - raise GenericConfigUpdaterError(f"Error resolving path: '{path}' due to incorrect ASIC number.") - scope = parts[0] - remainder = "/" + "/".join(parts[1:]) - elif parts[0] == HOST_NAMESPACE: - scope = HOST_NAMESPACE - remainder = "/" + "/".join(parts[1:]) +import json +import jsonpointer +import os +import subprocess + +from enum import Enum +from .gu_common import HOST_NAMESPACE, GenericConfigUpdaterError, EmptyTableError, ConfigWrapper, \ + DryRunConfigWrapper, PatchWrapper, genericUpdaterLogging +from .patch_sorter import StrictPatchSorter, NonStrictPatchSorter, ConfigSplitter, \ + TablesWithoutYangConfigSplitter, IgnorePathsFromYangConfigSplitter +from .change_applier import ChangeApplier, DryRunChangeApplier +from sonic_py_common import multi_asic + +CHECKPOINTS_DIR = "/etc/sonic/checkpoints" +CHECKPOINT_EXT = ".cp.json" + + +def extract_scope(path): + if not path: + raise Exception("Wrong patch with empty path.") + pointer = jsonpointer.JsonPointer(path) + parts = pointer.parts + if not parts: + raise GenericConfigUpdaterError("Wrong patch with empty path.") + if parts[0].startswith("asic"): + if not parts[0][len("asic"):].isnumeric(): + raise GenericConfigUpdaterError(f"Error resolving path: '{path}' due to incorrect ASIC number.") + scope = parts[0] + remainder = "/" + "/".join(parts[1:]) + elif parts[0] == HOST_NAMESPACE: + scope = HOST_NAMESPACE + remainder = "/" + "/".join(parts[1:]) + else: + scope = "" + remainder = path + return scope, remainder + + +def get_cmd_output(cmd): + proc = subprocess.Popen(cmd, text=True, stdout=subprocess.PIPE) + return proc.communicate()[0], proc.returncode + + +def get_config_json_by_namespace(scope): + cmd = ['sonic-cfggen', '-d', '--print-data'] + if scope is not None and scope != multi_asic.DEFAULT_NAMESPACE: + cmd += ['-n', scope] + stdout, rc = get_cmd_output(cmd) + if rc: + raise GenericConfigUpdaterError("Failed to get cmd output '{}':rc {}".format(cmd, rc)) + try: + config_json = json.loads(stdout) + except json.JSONDecodeError as e: + raise GenericConfigUpdaterError("Failed to get config by '{}' due to {}".format(cmd, e)) + return config_json + + +class ConfigLock: + def acquire_lock(self): + # TODO: Implement ConfigLock + pass + + def release_lock(self): + # TODO: Implement ConfigLock + pass + + +class ConfigFormat(Enum): + CONFIGDB = 1 + SONICYANG = 2 + + +class PatchApplier: + def __init__(self, + patchsorter=None, + changeapplier=None, + config_wrapper=None, + patch_wrapper=None, + scope=multi_asic.DEFAULT_NAMESPACE): + self.scope = scope + self.logger = genericUpdaterLogging.get_logger(title="Patch Applier", print_all_to_console=True) + self.config_wrapper = config_wrapper if config_wrapper is not None else ConfigWrapper(scope=self.scope) + self.patch_wrapper = patch_wrapper if patch_wrapper is not None else PatchWrapper(scope=self.scope) + self.patchsorter = patchsorter if patchsorter is not None else StrictPatchSorter(self.config_wrapper, self.patch_wrapper) + self.changeapplier = changeapplier if changeapplier is not None else ChangeApplier(scope=self.scope) + + def apply(self, patch, sort=True): + scope = self.scope if self.scope else HOST_NAMESPACE + self.logger.log_notice(f"{scope}: Patch application starting.") + self.logger.log_notice(f"{scope}: Patch: {patch}") + + # Get old config + self.logger.log_notice(f"{scope} getting current config db.") + old_config = self.config_wrapper.get_config_db_as_json() + + # Generate target config + self.logger.log_notice(f"{scope}: simulating the target full config after applying the patch.") + target_config = self.patch_wrapper.simulate_patch(patch, old_config) + + # Validate all JsonPatch operations on specified fields + self.logger.log_notice(f"{scope}: validating all JsonPatch operations are permitted on the specified fields") + self.config_wrapper.validate_field_operation(old_config, target_config) + + # Validate target config does not have empty tables since they do not show up in ConfigDb + self.logger.log_notice(f"""{scope}: validating target config does not have empty tables, + since they do not show up in ConfigDb.""") + empty_tables = self.config_wrapper.get_empty_tables(target_config) + if empty_tables: # if there are empty tables + empty_tables_txt = ", ".join(empty_tables) + raise EmptyTableError(f"{scope}: given patch is not valid because it will result in empty tables \ + which is not allowed in ConfigDb. \ + Table{'s' if len(empty_tables) != 1 else ''}: {empty_tables_txt}") + # Generate list of changes to apply + if sort: + self.logger.log_notice(f"{scope}: sorting patch updates.") + changes = self.patchsorter.sort(patch) else: - scope = "" - remainder = path + self.logger.log_notice(f"{scope}: converting patch to JsonChange.") + changes = [JsonChange(jsonpatch.JsonPatch([element])) for element in patch] - return scope, remainder + changes_len = len(changes) + self.logger.log_notice(f"The {scope} patch was converted into {changes_len} " \ + f"change{'s' if changes_len != 1 else ''}{':' if changes_len > 0 else '.'}") + # Apply changes in order + self.logger.log_notice(f"{scope}: applying {changes_len} change{'s' if changes_len != 1 else ''} " \ + f"in order{':' if changes_len > 0 else '.'}") + for change in changes: + self.logger.log_notice(f" * {change}") + self.changeapplier.apply(change) - def get_cmd_output(cmd): - proc = subprocess.Popen(cmd, text=True, stdout=subprocess.PIPE) - return proc.communicate()[0], proc.returncode + # Validate config updated successfully + self.logger.log_notice(f"{scope}: verifying patch updates are reflected on ConfigDB.") + new_config = self.config_wrapper.get_config_db_as_json() + self.changeapplier.remove_backend_tables_from_config(target_config) + self.changeapplier.remove_backend_tables_from_config(new_config) + if not (self.patch_wrapper.verify_same_json(target_config, new_config)): + raise GenericConfigUpdaterError(f"{scope}: after applying patch to config, there are still some parts not updated") + self.logger.log_notice(f"{scope} patch application completed.") - def get_config_json_by_namespace(scope): - cmd = ['sonic-cfggen', '-d', '--print-data'] - if scope is not None and scope != multi_asic.DEFAULT_NAMESPACE: - cmd += ['-n', scope] - stdout, rc = get_cmd_output(cmd) - if rc: - raise GenericConfigUpdaterError("Failed to get cmd output '{}':rc {}".format(cmd, rc)) +class ConfigReplacer: + def __init__(self, patch_applier=None, config_wrapper=None, patch_wrapper=None, scope=multi_asic.DEFAULT_NAMESPACE): + self.scope = scope + self.logger = genericUpdaterLogging.get_logger(title="Config Replacer", print_all_to_console=True) + self.patch_applier = patch_applier if patch_applier is not None else PatchApplier(scope=self.scope) + self.config_wrapper = config_wrapper if config_wrapper is not None else ConfigWrapper(scope=self.scope) + self.patch_wrapper = patch_wrapper if patch_wrapper is not None else PatchWrapper(scope=self.scope) - try: - config_json = json.loads(stdout) - except json.JSONDecodeError as e: - raise GenericConfigUpdaterError("Failed to get config by '{}' due to {}".format(cmd, e)) + def replace(self, target_config): + self.logger.log_notice("Config replacement starting.") + self.logger.log_notice(f"Target config length: {len(json.dumps(target_config))}.") - return config_json + self.logger.log_notice("Getting current config db.") + old_config = self.config_wrapper.get_config_db_as_json() + self.logger.log_notice("Generating patch between target config and current config db.") + patch = self.patch_wrapper.generate_patch(old_config, target_config) + self.logger.log_debug(f"Generated patch: {patch}.") # debug since the patch will printed again in 'patch_applier.apply' - class ConfigLock: - def acquire_lock(self): - # TODO: Implement ConfigLock - pass + self.logger.log_notice("Applying patch using 'Patch Applier'.") + self.patch_applier.apply(patch) - def release_lock(self): - # TODO: Implement ConfigLock - pass + self.logger.log_notice("Verifying config replacement is reflected on ConfigDB.") + new_config = self.config_wrapper.get_config_db_as_json() + if not (self.patch_wrapper.verify_same_json(target_config, new_config)): + raise GenericConfigUpdaterError(f"After replacing config, there is still some parts not updated") + self.logger.log_notice("Config replacement completed.") - class ConfigFormat(Enum): - CONFIGDB = 1 - SONICYANG = 2 - - class PatchApplier: - def __init__(self, - patchsorter=None, - changeapplier=None, - config_wrapper=None, - patch_wrapper=None, - scope=multi_asic.DEFAULT_NAMESPACE): - self.scope = scope - self.logger = genericUpdaterLogging.get_logger(title="Patch Applier", print_all_to_console=True) - self.config_wrapper = config_wrapper if config_wrapper is not None else ConfigWrapper(scope=self.scope) - self.patch_wrapper = patch_wrapper if patch_wrapper is not None else PatchWrapper(scope=self.scope) - self.patchsorter = patchsorter if patchsorter is not None else StrictPatchSorter(self.config_wrapper, self.patch_wrapper) - self.changeapplier = changeapplier if changeapplier is not None else ChangeApplier(scope=self.scope) - - def apply(self, patch, sort=True): - scope = self.scope if self.scope else HOST_NAMESPACE - self.logger.log_notice(f"{scope}: Patch application starting.") - self.logger.log_notice(f"{scope}: Patch: {patch}") - - # Get old config - self.logger.log_notice(f"{scope} getting current config db.") - old_config = self.config_wrapper.get_config_db_as_json() - - # Generate target config - self.logger.log_notice(f"{scope}: simulating the target full config after applying the patch.") - target_config = self.patch_wrapper.simulate_patch(patch, old_config) - - # Validate all JsonPatch operations on specified fields - self.logger.log_notice(f"{scope}: validating all JsonPatch operations are permitted on the specified fields") - self.config_wrapper.validate_field_operation(old_config, target_config) - - # Validate target config does not have empty tables since they do not show up in ConfigDb - self.logger.log_notice(f"""{scope}: validating target config does not have empty tables, - since they do not show up in ConfigDb.""") - - empty_tables = self.config_wrapper.get_empty_tables(target_config) - if empty_tables: # if there are empty tables - empty_tables_txt = ", ".join(empty_tables) - raise EmptyTableError(f"{scope}: given patch is not valid because it will result in empty tables \ - which is not allowed in ConfigDb. \ - Table{'s' if len(empty_tables) != 1 else ''}: {empty_tables_txt}") - - # Generate list of changes to apply - if sort: - self.logger.log_notice(f"{scope}: sorting patch updates.") - changes = self.patchsorter.sort(patch) - else: - self.logger.log_notice(f"{scope}: converting patch to JsonChange.") - changes = [JsonChange(jsonpatch.JsonPatch([element])) for element in patch] - changes_len = len(changes) - self.logger.log_notice(f"The {scope} patch was converted into {changes_len} " \ - f"change{'s' if changes_len != 1 else ''}{':' if changes_len > 0 else '.'}") +class FileSystemConfigRollbacker: + def __init__(self, + checkpoints_dir=CHECKPOINTS_DIR, + config_replacer=None, + config_wrapper=None, + scope=multi_asic.DEFAULT_NAMESPACE): + self.scope = scope + self.logger = genericUpdaterLogging.get_logger(title="Config Rollbacker", print_all_to_console=True) + self.util = Util(checkpoints_dir=checkpoints_dir) + self.checkpoints_dir = checkpoints_dir + self.config_replacer = config_replacer if config_replacer is not None else ConfigReplacer(scope=self.scope) + self.config_wrapper = config_wrapper if config_wrapper is not None else ConfigWrapper(scope=self.scope) - # Apply changes in order - self.logger.log_notice(f"{scope}: applying {changes_len} change{'s' if changes_len != 1 else ''} " \ - f"in order{':' if changes_len > 0 else '.'}") - for change in changes: - self.logger.log_notice(f" * {change}") - self.changeapplier.apply(change) + def rollback(self, checkpoint_name): + self.logger.log_notice("Config rollbacking starting.") + self.logger.log_notice(f"Checkpoint name: {checkpoint_name}.") - # Validate config updated successfully - self.logger.log_notice(f"{scope}: verifying patch updates are reflected on ConfigDB.") - new_config = self.config_wrapper.get_config_db_as_json() - self.changeapplier.remove_backend_tables_from_config(target_config) - self.changeapplier.remove_backend_tables_from_config(new_config) - if not(self.patch_wrapper.verify_same_json(target_config, new_config)): - raise GenericConfigUpdaterError(f"{scope}: after applying patch to config, there are still some parts not updated") + self.logger.log_notice(f"Verifying '{checkpoint_name}' exists.") + if not self.util.check_checkpoint_exists(checkpoint_name): + raise ValueError(f"Checkpoint '{checkpoint_name}' does not exist") - self.logger.log_notice(f"{scope} patch application completed.") + self.logger.log_notice(f"Loading checkpoint into memory.") + target_config = self.util.get_checkpoint_content(checkpoint_name) + self.logger.log_notice(f"Replacing config using 'Config Replacer'.") + self.config_replacer.replace(target_config) - class ConfigReplacer: - def __init__(self, patch_applier=None, config_wrapper=None, patch_wrapper=None, scope=multi_asic.DEFAULT_NAMESPACE): - self.scope = scope - self.logger = genericUpdaterLogging.get_logger(title="Config Replacer", print_all_to_console=True) - self.patch_applier = patch_applier if patch_applier is not None else PatchApplier(scope=self.scope) - self.config_wrapper = config_wrapper if config_wrapper is not None else ConfigWrapper(scope=self.scope) - self.patch_wrapper = patch_wrapper if patch_wrapper is not None else PatchWrapper(scope=self.scope) + self.logger.log_notice("Config rollbacking completed.") - def replace(self, target_config): - self.logger.log_notice("Config replacement starting.") - self.logger.log_notice(f"Target config length: {len(json.dumps(target_config))}.") + def checkpoint(self, checkpoint_name): + self.logger.log_notice("Config checkpoint starting.") + self.logger.log_notice(f"Checkpoint name: {checkpoint_name}.") - self.logger.log_notice("Getting current config db.") - old_config = self.config_wrapper.get_config_db_as_json() + self.logger.log_notice("Getting current config db.") + json_content = self.config_wrapper.get_config_db_as_json() - self.logger.log_notice("Generating patch between target config and current config db.") - patch = self.patch_wrapper.generate_patch(old_config, target_config) - self.logger.log_debug(f"Generated patch: {patch}.") # debug since the patch will printed again in 'patch_applier.apply' + self.logger.log_notice("Getting checkpoint full-path.") + path = self.util.get_checkpoint_full_path(checkpoint_name) - self.logger.log_notice("Applying patch using 'Patch Applier'.") - self.patch_applier.apply(patch) + self.logger.log_notice("Ensuring checkpoint directory exist.") + self.util.ensure_checkpoints_dir_exists() - self.logger.log_notice("Verifying config replacement is reflected on ConfigDB.") - new_config = self.config_wrapper.get_config_db_as_json() - if not(self.patch_wrapper.verify_same_json(target_config, new_config)): - raise GenericConfigUpdaterError(f"After replacing config, there is still some parts not updated") + self.logger.log_notice(f"Saving config db content to {path}.") + self.util.save_json_file(path, json_content) - self.logger.log_notice("Config replacement completed.") + self.logger.log_notice("Config checkpoint completed.") + def list_checkpoints(self): + self.logger.log_info("Listing checkpoints starting.") - class FileSystemConfigRollbacker: - def __init__(self, - checkpoints_dir=CHECKPOINTS_DIR, - config_replacer=None, - config_wrapper=None, - scope=multi_asic.DEFAULT_NAMESPACE): - self.scope = scope - self.logger = genericUpdaterLogging.get_logger(title="Config Rollbacker", print_all_to_console=True) - self.util = Util(checkpoints_dir=checkpoints_dir) - self.checkpoints_dir = checkpoints_dir - self.config_replacer = config_replacer if config_replacer is not None else ConfigReplacer(scope=self.scope) - self.config_wrapper = config_wrapper if config_wrapper is not None else ConfigWrapper(scope=self.scope) + self.logger.log_info(f"Verifying checkpoints directory '{self.checkpoints_dir}' exists.") + if not self.util.checkpoints_dir_exist(): + self.logger.log_info("Checkpoints directory is empty, returning empty checkpoints list.") + return [] - def rollback(self, checkpoint_name): - self.logger.log_notice("Config rollbacking starting.") - self.logger.log_notice(f"Checkpoint name: {checkpoint_name}.") - - self.logger.log_notice(f"Verifying '{checkpoint_name}' exists.") - if not self.util.check_checkpoint_exists(checkpoint_name): - raise ValueError(f"Checkpoint '{checkpoint_name}' does not exist") - - self.logger.log_notice(f"Loading checkpoint into memory.") - target_config = self.util.get_checkpoint_content(checkpoint_name) - - self.logger.log_notice(f"Replacing config using 'Config Replacer'.") - self.config_replacer.replace(target_config) - - self.logger.log_notice("Config rollbacking completed.") - - def checkpoint(self, checkpoint_name): - self.logger.log_notice("Config checkpoint starting.") - self.logger.log_notice(f"Checkpoint name: {checkpoint_name}.") - - self.logger.log_notice("Getting current config db.") - json_content = self.config_wrapper.get_config_db_as_json() - - self.logger.log_notice("Getting checkpoint full-path.") - path = self.util.get_checkpoint_full_path(checkpoint_name) - - self.logger.log_notice("Ensuring checkpoint directory exist.") - self.util.ensure_checkpoints_dir_exists() - - self.logger.log_notice(f"Saving config db content to {path}.") - self.util.save_json_file(path, json_content) - - self.logger.log_notice("Config checkpoint completed.") - - def list_checkpoints(self): - self.logger.log_info("Listing checkpoints starting.") - - self.logger.log_info(f"Verifying checkpoints directory '{self.checkpoints_dir}' exists.") - if not self.util.checkpoints_dir_exist(): - self.logger.log_info("Checkpoints directory is empty, returning empty checkpoints list.") - return [] - - self.logger.log_info("Getting checkpoints in checkpoints directory.") - checkpoint_names = self.util.get_checkpoint_names() - - checkpoints_len = len(checkpoint_names) - self.logger.log_info(f"Found {checkpoints_len} checkpoint{'s' if checkpoints_len != 1 else ''}{':' if checkpoints_len > 0 else '.'}") - for checkpoint_name in checkpoint_names: - self.logger.log_info(f" * {checkpoint_name}") - - self.logger.log_info("Listing checkpoints completed.") - - return checkpoint_names - - def delete_checkpoint(self, checkpoint_name): - self.logger.log_notice("Deleting checkpoint starting.") - self.logger.log_notice(f"Checkpoint name: {checkpoint_name}.") - - self.logger.log_notice(f"Checking checkpoint exists.") - if not self.util.check_checkpoint_exists(checkpoint_name): - raise ValueError(f"Checkpoint '{checkpoint_name}' does not exist") - - self.logger.log_notice(f"Deleting checkpoint.") - self.util.delete_checkpoint(checkpoint_name) - - self.logger.log_notice("Deleting checkpoint completed.") - - - class MultiASICConfigReplacer: - def __init__(self, checkpoints_dir=CHECKPOINTS_DIR): - self.logger = genericUpdaterLogging.get_logger(title="MultiASICConfigReplacer", - print_all_to_console=True) - self.scopelist = [HOST_NAMESPACE].extend(multi_asic.get_namespace_list()) - self.checkpoints_dir = checkpoints_dir - self.util = Util(checkpoints_dir=checkpoints_dir) - - def replace(self, target_config): - config_keys = set(target_config.keys()) - missing_scopes = set(self.scopelist) - config_keys - if missing_scopes: - raise GenericConfigUpdaterError(f"To be replace config is missing scope: {missing_scopes}") - - for scope in self.scopelist: - scope_config = target_config.pop(scope) - if scope.lower() == HOST_NAMESPACE: - scope = multi_asic.DEFAULT_NAMESPACE - ConfigReplacer(namespace=scope).replace(scope_config) - - - class MultiASICConfigRollbacker(FileSystemConfigRollbacker): - def __init__(self, checkpoints_dir=CHECKPOINTS_DIR): - self.logger = genericUpdaterLogging.get_logger(title="MultiASICConfigRollbacker", - print_all_to_console=True) - self.scopelist = [HOST_NAMESPACE].extend(multi_asic.get_namespace_list()) - self.checkpoints_dir = checkpoints_dir - self.util = Util(checkpoints_dir=checkpoints_dir) - - def rollback(self, checkpoint_name): - self.logger.log_notice("Config rollbacking starting.") - self.logger.log_notice(f"Checkpoint name: {checkpoint_name}.") - self.logger.log_notice(f"Verifying '{checkpoint_name}' exists.") - if not self.util.check_checkpoint_exists(checkpoint_name): - raise ValueError(f"Checkpoint '{checkpoint_name}' does not exist") - self.logger.log_notice(f"Loading checkpoint '{checkpoint_name}' into memory.") - target_config = self.util.get_checkpoint_content(checkpoint_name) - self.logger.log_notice(f"Replacing config '{checkpoint_name}' using 'Config Replacer'.") - for scope in self.scopelist: - if scope.lower() == multi_asic.DEFAULT_NAMESPACE: - config = target_config.pop(HOST_NAMESPACE) - else: - config = target_config.pop(scope) - ConfigReplacer(namespace=scope).replace(config) - self.logger.log_notice("Config rollbacking completed.") + self.logger.log_info("Getting checkpoints in checkpoints directory.") + checkpoint_names = self.util.get_checkpoint_names() - def checkpoint(self, checkpoint_name): - all_configs = {} - self.logger.log_notice("Config checkpoint starting.") - self.logger.log_notice(f"Checkpoint name: {checkpoint_name}.") - for scope in self.scopelist: - self.logger.log_notice(f"Getting current {scope} config db.") - config = get_config_json_by_namespace(scope) - if scope.lower() == multi_asic.DEFAULT_NAMESPACE: - scope = HOST_NAMESPACE - all_configs[scope] = config + checkpoints_len = len(checkpoint_names) + self.logger.log_info(f"Found {checkpoints_len} checkpoint{'s' if checkpoints_len != 1 else ''}{':' if checkpoints_len > 0 else '.'}") + for checkpoint_name in checkpoint_names: + self.logger.log_info(f" * {checkpoint_name}") - self.logger.log_notice("Getting checkpoint full-path.") - path = self.util.get_checkpoint_full_path(checkpoint_name) - self.logger.log_notice("Ensuring checkpoint directory exist.") - self.util.ensure_checkpoints_dir_exists() - self.logger.log_notice(f"Saving config db content to {path}.") - - self.util.save_json_file(path, all_configs) - self.logger.log_notice("Config checkpoint completed.") - - - class Util: - def __init__(self, checkpoints_dir=CHECKPOINTS_DIR): - self.checkpoints_dir = checkpoints_dir - - def ensure_checkpoints_dir_exists(self): - os.makedirs(self.checkpoints_dir, exist_ok=True) - - def save_json_file(self, path, json_content): - with open(path, "w") as fh: - fh.write(json.dumps(json_content)) - - def get_checkpoint_content(self, checkpoint_name): - path = self.get_checkpoint_full_path(checkpoint_name) - with open(path) as fh: - text = fh.read() - return json.loads(text) - - def get_checkpoint_full_path(self, name): - return os.path.join(self.checkpoints_dir, f"{name}{CHECKPOINT_EXT}") - - def get_checkpoint_names(self): - file_names = [] - for file_name in os.listdir(self.checkpoints_dir): - if file_name.endswith(CHECKPOINT_EXT): - # Remove extension from file name. - # Example assuming ext is '.cp.json', then 'checkpoint1.cp.json' becomes 'checkpoint1' - file_names.append(file_name[:-len(CHECKPOINT_EXT)]) - - return file_names - - def checkpoints_dir_exist(self): - return os.path.isdir(self.checkpoints_dir) - - def check_checkpoint_exists(self, name): - path = self.get_checkpoint_full_path(name) - return os.path.isfile(path) - - def delete_checkpoint(self, name): - path = self.get_checkpoint_full_path(name) - return os.remove(path) - - - class Decorator(PatchApplier, ConfigReplacer, FileSystemConfigRollbacker): - def __init__(self, decorated_patch_applier=None, decorated_config_replacer=None, decorated_config_rollbacker=None, scope=multi_asic.DEFAULT_NAMESPACE): - # initing base classes to make LGTM happy - PatchApplier.__init__(self, scope=scope) - ConfigReplacer.__init__(self, scope=scope) - FileSystemConfigRollbacker.__init__(self, scope=scope) - - self.decorated_patch_applier = decorated_patch_applier - self.decorated_config_replacer = decorated_config_replacer - self.decorated_config_rollbacker = decorated_config_rollbacker - - def apply(self, patch): - self.decorated_patch_applier.apply(patch) - - def replace(self, target_config): - self.decorated_config_replacer.replace(target_config) - - def rollback(self, checkpoint_name): - self.decorated_config_rollbacker.rollback(checkpoint_name) - - def checkpoint(self, checkpoint_name): - self.decorated_config_rollbacker.checkpoint(checkpoint_name) - - def list_checkpoints(self): - return self.decorated_config_rollbacker.list_checkpoints() - - def delete_checkpoint(self, checkpoint_name): - self.decorated_config_rollbacker.delete_checkpoint(checkpoint_name) - - - class SonicYangDecorator(Decorator): - def __init__(self, patch_wrapper, config_wrapper, decorated_patch_applier=None, decorated_config_replacer=None, scope=multi_asic.DEFAULT_NAMESPACE): - Decorator.__init__(self, decorated_patch_applier, decorated_config_replacer, scope=scope) - - self.scope = scope - self.patch_wrapper = patch_wrapper - self.config_wrapper = config_wrapper - - def apply(self, patch): - config_db_patch = self.patch_wrapper.convert_sonic_yang_patch_to_config_db_patch(patch) - Decorator.apply(self, config_db_patch) + self.logger.log_info("Listing checkpoints completed.") - def replace(self, target_config): - config_db_target_config = self.config_wrapper.convert_sonic_yang_to_config_db(target_config) - Decorator.replace(self, config_db_target_config) + return checkpoint_names + def delete_checkpoint(self, checkpoint_name): + self.logger.log_notice("Deleting checkpoint starting.") + self.logger.log_notice(f"Checkpoint name: {checkpoint_name}.") - class ConfigLockDecorator(Decorator): - def __init__(self, - decorated_patch_applier=None, - decorated_config_replacer=None, - decorated_config_rollbacker=None, - config_lock=ConfigLock(), - scope=multi_asic.DEFAULT_NAMESPACE): - Decorator.__init__(self, decorated_patch_applier, decorated_config_replacer, decorated_config_rollbacker, scope=scope) + self.logger.log_notice(f"Checking checkpoint exists.") + if not self.util.check_checkpoint_exists(checkpoint_name): + raise ValueError(f"Checkpoint '{checkpoint_name}' does not exist") - self.config_lock = config_lock + self.logger.log_notice(f"Deleting checkpoint.") + self.util.delete_checkpoint(checkpoint_name) - def apply(self, patch, sort=True): - self.execute_write_action(Decorator.apply, self, patch) + self.logger.log_notice("Deleting checkpoint completed.") - def replace(self, target_config): - self.execute_write_action(Decorator.replace, self, target_config) - def rollback(self, checkpoint_name): - self.execute_write_action(Decorator.rollback, self, checkpoint_name) +class MultiASICConfigReplacer: + def __init__(self, checkpoints_dir=CHECKPOINTS_DIR): + self.logger = genericUpdaterLogging.get_logger(title="MultiASICConfigReplacer", + print_all_to_console=True) + self.scopelist = [HOST_NAMESPACE].extend(multi_asic.get_namespace_list()) + self.checkpoints_dir = checkpoints_dir + self.util = Util(checkpoints_dir=checkpoints_dir) - def checkpoint(self, checkpoint_name): - self.execute_write_action(Decorator.checkpoint, self, checkpoint_name) + def replace(self, target_config): + config_keys = set(target_config.keys()) + missing_scopes = set(self.scopelist) - config_keys + if missing_scopes: + raise GenericConfigUpdaterError(f"To be replace config is missing scope: {missing_scopes}") - def execute_write_action(self, action, *args): - self.config_lock.acquire_lock() - action(*args) - self.config_lock.release_lock() + for scope in self.scopelist: + scope_config = target_config.pop(scope) + if scope.lower() == HOST_NAMESPACE: + scope = multi_asic.DEFAULT_NAMESPACE + ConfigReplacer(namespace=scope).replace(scope_config) - class GenericUpdateFactory: - def __init__(self, scope=multi_asic.DEFAULT_NAMESPACE): - self.scope = scope +class MultiASICConfigRollbacker(FileSystemConfigRollbacker): + def __init__(self, checkpoints_dir=CHECKPOINTS_DIR): + self.logger = genericUpdaterLogging.get_logger(title="MultiASICConfigRollbacker", + print_all_to_console=True) + self.scopelist = [HOST_NAMESPACE].extend(multi_asic.get_namespace_list()) + self.checkpoints_dir = checkpoints_dir + self.util = Util(checkpoints_dir=checkpoints_dir) - def create_patch_applier(self, config_format, verbose, dry_run, ignore_non_yang_tables, ignore_paths): - self.init_verbose_logging(verbose) - config_wrapper = self.get_config_wrapper(dry_run) - change_applier = self.get_change_applier(dry_run, config_wrapper) - patch_wrapper = PatchWrapper(config_wrapper, scope=self.scope) - patch_sorter = self.get_patch_sorter(ignore_non_yang_tables, ignore_paths, config_wrapper, patch_wrapper) - patch_applier = PatchApplier(config_wrapper=config_wrapper, - patchsorter=patch_sorter, - patch_wrapper=patch_wrapper, - changeapplier=change_applier, - scope=self.scope) + def rollback(self, checkpoint_name): + self.logger.log_notice("Config rollbacking starting.") + self.logger.log_notice(f"Checkpoint name: {checkpoint_name}.") + self.logger.log_notice(f"Verifying '{checkpoint_name}' exists.") - if config_format == ConfigFormat.CONFIGDB: - pass - elif config_format == ConfigFormat.SONICYANG: - patch_applier = SonicYangDecorator(decorated_patch_applier=patch_applier, - patch_wrapper=patch_wrapper, - config_wrapper=config_wrapper, - scope=self.scope) - else: - raise ValueError(f"config-format '{config_format}' is not supported") - - if not dry_run: - patch_applier = ConfigLockDecorator(decorated_patch_applier=patch_applier, scope=self.scope) - - return patch_applier - - def create_config_replacer(self, config_format, verbose, dry_run, ignore_non_yang_tables, ignore_paths): - self.init_verbose_logging(verbose) - - config_wrapper = self.get_config_wrapper(dry_run) - change_applier = self.get_change_applier(dry_run, config_wrapper) - patch_wrapper = PatchWrapper(config_wrapper, scope=self.scope) - patch_sorter = self.get_patch_sorter(ignore_non_yang_tables, ignore_paths, config_wrapper, patch_wrapper) - patch_applier = PatchApplier(config_wrapper=config_wrapper, - patchsorter=patch_sorter, - patch_wrapper=patch_wrapper, - changeapplier=change_applier, - scope=self.scope) - - config_replacer = MultiASICConfigReplacer() if multi_asic.is_multi_asic() else ConfigReplacer( - patch_applier=patch_applier, config_wrapper=config_wrapper, scope=self.scope) - - if config_format == ConfigFormat.CONFIGDB: - pass - elif config_format == ConfigFormat.SONICYANG: - config_replacer = SonicYangDecorator(decorated_config_replacer=config_replacer, - patch_wrapper=patch_wrapper, - config_wrapper=config_wrapper, - scope=self.scope) + if not self.util.check_checkpoint_exists(checkpoint_name): + raise ValueError(f"Checkpoint '{checkpoint_name}' does not exist") + + self.logger.log_notice(f"Loading checkpoint '{checkpoint_name}' into memory.") + target_config = self.util.get_checkpoint_content(checkpoint_name) + self.logger.log_notice(f"Replacing config '{checkpoint_name}' using 'Config Replacer'.") + + for scope in self.scopelist: + if scope.lower() == multi_asic.DEFAULT_NAMESPACE: + config = target_config.pop(HOST_NAMESPACE) else: - raise ValueError(f"config-format '{config_format}' is not supported") + config = target_config.pop(scope) + ConfigReplacer(namespace=scope).replace(config) - if not dry_run: - config_replacer = ConfigLockDecorator(decorated_config_replacer=config_replacer, scope=self.scope) + self.logger.log_notice("Config rollbacking completed.") - return config_replacer + def checkpoint(self, checkpoint_name): + all_configs = {} + self.logger.log_notice("Config checkpoint starting.") + self.logger.log_notice(f"Checkpoint name: {checkpoint_name}.") - def create_config_rollbacker(self, verbose, dry_run=False, ignore_non_yang_tables=False, ignore_paths=[]): - self.init_verbose_logging(verbose) + for scope in self.scopelist: + self.logger.log_notice(f"Getting current {scope} config db.") + config = get_config_json_by_namespace(scope) + if scope.lower() == multi_asic.DEFAULT_NAMESPACE: + scope = HOST_NAMESPACE + all_configs[scope] = config - config_wrapper = self.get_config_wrapper(dry_run) - change_applier = self.get_change_applier(dry_run, config_wrapper) - patch_wrapper = PatchWrapper(config_wrapper, scope=self.scope) - patch_sorter = self.get_patch_sorter(ignore_non_yang_tables, ignore_paths, config_wrapper, patch_wrapper) - patch_applier = PatchApplier(config_wrapper=config_wrapper, - patchsorter=patch_sorter, - patch_wrapper=patch_wrapper, - changeapplier=change_applier, - scope=self.scope) + self.logger.log_notice("Getting checkpoint full-path.") + path = self.util.get_checkpoint_full_path(checkpoint_name) - config_replacer = MultiASICConfigReplacer() if multi_asic.is_multi_asic() else ConfigReplacer( - config_wrapper=config_wrapper, patch_applier=patch_applier, scope=self.scope) - config_rollbacker = MultiASICConfigRollbacker() if multi_asic.is_multi_asic() else FileSystemConfigRollbacker( - config_wrapper=config_wrapper, config_replacer=config_replacer, scope=self.scope) + self.logger.log_notice("Ensuring checkpoint directory exist.") + self.util.ensure_checkpoints_dir_exists() - if not dry_run: - config_rollbacker = ConfigLockDecorator(decorated_config_rollbacker=config_rollbacker, scope=self.scope) + self.logger.log_notice(f"Saving config db content to {path}.") + self.util.save_json_file(path, all_configs) - return config_rollbacker + self.logger.log_notice("Config checkpoint completed.") - def init_verbose_logging(self, verbose): - genericUpdaterLogging.set_verbose(verbose) - def get_config_wrapper(self, dry_run): - if dry_run: - return DryRunConfigWrapper(scope=self.scope) - else: - return ConfigWrapper(scope=self.scope) +class Util: + def __init__(self, checkpoints_dir=CHECKPOINTS_DIR): + self.checkpoints_dir = checkpoints_dir - def get_change_applier(self, dry_run, config_wrapper): - if dry_run: - return DryRunChangeApplier(config_wrapper) - else: - return ChangeApplier(scope=self.scope) + def ensure_checkpoints_dir_exists(self): + os.makedirs(self.checkpoints_dir, exist_ok=True) + + def save_json_file(self, path, json_content): + with open(path, "w") as fh: + fh.write(json.dumps(json_content)) + + def get_checkpoint_content(self, checkpoint_name): + path = self.get_checkpoint_full_path(checkpoint_name) + with open(path) as fh: + text = fh.read() + return json.loads(text) + + def get_checkpoint_full_path(self, name): + return os.path.join(self.checkpoints_dir, f"{name}{CHECKPOINT_EXT}") + + def get_checkpoint_names(self): + file_names = [] + for file_name in os.listdir(self.checkpoints_dir): + if file_name.endswith(CHECKPOINT_EXT): + # Remove extension from file name. + # Example assuming ext is '.cp.json', then 'checkpoint1.cp.json' becomes 'checkpoint1' + file_names.append(file_name[:-len(CHECKPOINT_EXT)]) + return file_names + + def checkpoints_dir_exist(self): + return os.path.isdir(self.checkpoints_dir) + + def check_checkpoint_exists(self, name): + path = self.get_checkpoint_full_path(name) + return os.path.isfile(path) + + def delete_checkpoint(self, name): + path = self.get_checkpoint_full_path(name) + return os.remove(path) + + +class Decorator(PatchApplier, ConfigReplacer, FileSystemConfigRollbacker): + def __init__(self, decorated_patch_applier=None, decorated_config_replacer=None, decorated_config_rollbacker=None, scope=multi_asic.DEFAULT_NAMESPACE): + # initing base classes to make LGTM happy + PatchApplier.__init__(self, scope=scope) + ConfigReplacer.__init__(self, scope=scope) + FileSystemConfigRollbacker.__init__(self, scope=scope) + self.decorated_patch_applier = decorated_patch_applier + self.decorated_config_replacer = decorated_config_replacer + self.decorated_config_rollbacker = decorated_config_rollbacker + + def apply(self, patch): + self.decorated_patch_applier.apply(patch) + + def replace(self, target_config): + self.decorated_config_replacer.replace(target_config) + + def rollback(self, checkpoint_name): + self.decorated_config_rollbacker.rollback(checkpoint_name) + + def checkpoint(self, checkpoint_name): + self.decorated_config_rollbacker.checkpoint(checkpoint_name) + + def list_checkpoints(self): + return self.decorated_config_rollbacker.list_checkpoints() + + def delete_checkpoint(self, checkpoint_name): + self.decorated_config_rollbacker.delete_checkpoint(checkpoint_name) + + +class SonicYangDecorator(Decorator): + def __init__(self, patch_wrapper, config_wrapper, decorated_patch_applier=None, decorated_config_replacer=None, scope=multi_asic.DEFAULT_NAMESPACE): + Decorator.__init__(self, decorated_patch_applier, decorated_config_replacer, scope=scope) + self.scope = scope + self.patch_wrapper = patch_wrapper + self.config_wrapper = config_wrapper + + def apply(self, patch): + config_db_patch = self.patch_wrapper.convert_sonic_yang_patch_to_config_db_patch(patch) + Decorator.apply(self, config_db_patch) + + def replace(self, target_config): + config_db_target_config = self.config_wrapper.convert_sonic_yang_to_config_db(target_config) + Decorator.replace(self, config_db_target_config) + + +class ConfigLockDecorator(Decorator): + def __init__(self, + decorated_patch_applier=None, + decorated_config_replacer=None, + decorated_config_rollbacker=None, + config_lock=ConfigLock(), + scope=multi_asic.DEFAULT_NAMESPACE): + Decorator.__init__(self, decorated_patch_applier, decorated_config_replacer, decorated_config_rollbacker, scope=scope) + self.config_lock = config_lock + + def apply(self, patch, sort=True): + self.execute_write_action(Decorator.apply, self, patch) + + def replace(self, target_config): + self.execute_write_action(Decorator.replace, self, target_config) + + def rollback(self, checkpoint_name): + self.execute_write_action(Decorator.rollback, self, checkpoint_name) + + def checkpoint(self, checkpoint_name): + self.execute_write_action(Decorator.checkpoint, self, checkpoint_name) + + def execute_write_action(self, action, *args): + self.config_lock.acquire_lock() + action(*args) + self.config_lock.release_lock() + + +class GenericUpdateFactory: + def __init__(self, scope=multi_asic.DEFAULT_NAMESPACE): + self.scope = scope + + def create_patch_applier(self, config_format, verbose, dry_run, ignore_non_yang_tables, ignore_paths): + self.init_verbose_logging(verbose) + config_wrapper = self.get_config_wrapper(dry_run) + change_applier = self.get_change_applier(dry_run, config_wrapper) + patch_wrapper = PatchWrapper(config_wrapper, scope=self.scope) + patch_sorter = self.get_patch_sorter(ignore_non_yang_tables, ignore_paths, config_wrapper, patch_wrapper) + patch_applier = PatchApplier(config_wrapper=config_wrapper, + patchsorter=patch_sorter, + patch_wrapper=patch_wrapper, + changeapplier=change_applier, + scope=self.scope) + + if config_format == ConfigFormat.CONFIGDB: + pass + elif config_format == ConfigFormat.SONICYANG: + patch_applier = SonicYangDecorator(decorated_patch_applier=patch_applier, + patch_wrapper=patch_wrapper, + config_wrapper=config_wrapper, + scope=self.scope) + else: + raise ValueError(f"config-format '{config_format}' is not supported") + + if not dry_run: + patch_applier = ConfigLockDecorator(decorated_patch_applier=patch_applier, scope=self.scope) + + return patch_applier + + def create_config_replacer(self, config_format, verbose, dry_run, ignore_non_yang_tables, ignore_paths): + self.init_verbose_logging(verbose) + config_wrapper = self.get_config_wrapper(dry_run) + change_applier = self.get_change_applier(dry_run, config_wrapper) + patch_wrapper = PatchWrapper(config_wrapper, scope=self.scope) + patch_sorter = self.get_patch_sorter(ignore_non_yang_tables, ignore_paths, config_wrapper, patch_wrapper) + patch_applier = PatchApplier(config_wrapper=config_wrapper, + patchsorter=patch_sorter, + patch_wrapper=patch_wrapper, + changeapplier=change_applier, + scope=self.scope) + config_replacer = MultiASICConfigReplacer() if multi_asic.is_multi_asic() else ConfigReplacer( + patch_applier=patch_applier, config_wrapper=config_wrapper, scope=self.scope) + + if config_format == ConfigFormat.CONFIGDB: + pass + elif config_format == ConfigFormat.SONICYANG: + config_replacer = SonicYangDecorator(decorated_config_replacer=config_replacer, + patch_wrapper=patch_wrapper, + config_wrapper=config_wrapper, + scope=self.scope) + else: + raise ValueError(f"config-format '{config_format}' is not supported") + + if not dry_run: + config_replacer = ConfigLockDecorator(decorated_config_replacer=config_replacer, scope=self.scope) + + return config_replacer + + def create_config_rollbacker(self, verbose, dry_run=False, ignore_non_yang_tables=False, ignore_paths=[]): + self.init_verbose_logging(verbose) + config_wrapper = self.get_config_wrapper(dry_run) + change_applier = self.get_change_applier(dry_run, config_wrapper) + patch_wrapper = PatchWrapper(config_wrapper, scope=self.scope) + patch_sorter = self.get_patch_sorter(ignore_non_yang_tables, ignore_paths, config_wrapper, patch_wrapper) + patch_applier = PatchApplier(config_wrapper=config_wrapper, + patchsorter=patch_sorter, + patch_wrapper=patch_wrapper, + changeapplier=change_applier, + scope=self.scope) + config_replacer = MultiASICConfigReplacer() if multi_asic.is_multi_asic() else ConfigReplacer( + config_wrapper=config_wrapper, patch_applier=patch_applier, scope=self.scope) + config_rollbacker = MultiASICConfigRollbacker() if multi_asic.is_multi_asic() else FileSystemConfigRollbacker( + config_wrapper=config_wrapper, config_replacer=config_replacer, scope=self.scope) + + if not dry_run: + config_rollbacker = ConfigLockDecorator(decorated_config_rollbacker=config_rollbacker, scope=self.scope) + + return config_rollbacker + + def init_verbose_logging(self, verbose): + genericUpdaterLogging.set_verbose(verbose) + + def get_config_wrapper(self, dry_run): + if dry_run: + return DryRunConfigWrapper(scope=self.scope) + else: + return ConfigWrapper(scope=self.scope) + + def get_change_applier(self, dry_run, config_wrapper): + if dry_run: + return DryRunChangeApplier(config_wrapper) + else: + return ChangeApplier(scope=self.scope) - def get_patch_sorter(self, ignore_non_yang_tables, ignore_paths, config_wrapper, patch_wrapper): - if not ignore_non_yang_tables and not ignore_paths: - return StrictPatchSorter(config_wrapper, patch_wrapper) + def get_patch_sorter(self, ignore_non_yang_tables, ignore_paths, config_wrapper, patch_wrapper): + if not ignore_non_yang_tables and not ignore_paths: + return StrictPatchSorter(config_wrapper, patch_wrapper) - inner_config_splitters = [] - if ignore_non_yang_tables: - inner_config_splitters.append(TablesWithoutYangConfigSplitter(config_wrapper)) + inner_config_splitters = [] + if ignore_non_yang_tables: + inner_config_splitters.append(TablesWithoutYangConfigSplitter(config_wrapper)) - if ignore_paths: - inner_config_splitters.append(IgnorePathsFromYangConfigSplitter(ignore_paths, config_wrapper)) + if ignore_paths: + inner_config_splitters.append(IgnorePathsFromYangConfigSplitter(ignore_paths, config_wrapper)) - config_splitter = ConfigSplitter(config_wrapper, inner_config_splitters) + config_splitter = ConfigSplitter(config_wrapper, inner_config_splitters) - return NonStrictPatchSorter(config_wrapper, patch_wrapper, config_splitter) + return NonStrictPatchSorter(config_wrapper, patch_wrapper, config_splitter) - class GenericUpdater: - def __init__(self, generic_update_factory=None, scope=multi_asic.DEFAULT_NAMESPACE): - self.generic_update_factory = \ - generic_update_factory if generic_update_factory is not None else GenericUpdateFactory(scope=scope) +class GenericUpdater: + def __init__(self, generic_update_factory=None, scope=multi_asic.DEFAULT_NAMESPACE): + self.generic_update_factory = \ + generic_update_factory if generic_update_factory is not None else GenericUpdateFactory(scope=scope) - def apply_patch(self, patch, config_format, verbose, dry_run, ignore_non_yang_tables, ignore_paths, sort=True): - patch_applier = self.generic_update_factory.create_patch_applier(config_format, verbose, dry_run, ignore_non_yang_tables, ignore_paths) - patch_applier.apply(patch, sort) + def apply_patch(self, patch, config_format, verbose, dry_run, ignore_non_yang_tables, ignore_paths, sort=True): + patch_applier = self.generic_update_factory.create_patch_applier(config_format, verbose, dry_run, ignore_non_yang_tables, ignore_paths) + patch_applier.apply(patch, sort) - def replace(self, target_config, config_format, verbose, dry_run, ignore_non_yang_tables, ignore_paths): - config_replacer = self.generic_update_factory.create_config_replacer(config_format, verbose, dry_run, ignore_non_yang_tables, ignore_paths) - config_replacer.replace(target_config) + def replace(self, target_config, config_format, verbose, dry_run, ignore_non_yang_tables, ignore_paths): + config_replacer = self.generic_update_factory.create_config_replacer(config_format, verbose, dry_run, ignore_non_yang_tables, ignore_paths) + config_replacer.replace(target_config) - def rollback(self, checkpoint_name, verbose, dry_run, ignore_non_yang_tables, ignore_paths): - config_rollbacker = self.generic_update_factory.create_config_rollbacker(verbose, dry_run, ignore_non_yang_tables, ignore_paths) - config_rollbacker.rollback(checkpoint_name) + def rollback(self, checkpoint_name, verbose, dry_run, ignore_non_yang_tables, ignore_paths): + config_rollbacker = self.generic_update_factory.create_config_rollbacker(verbose, dry_run, ignore_non_yang_tables, ignore_paths) + config_rollbacker.rollback(checkpoint_name) - def checkpoint(self, checkpoint_name, verbose): - config_rollbacker = self.generic_update_factory.create_config_rollbacker(verbose) - config_rollbacker.checkpoint(checkpoint_name) + def checkpoint(self, checkpoint_name, verbose): + config_rollbacker = self.generic_update_factory.create_config_rollbacker(verbose) + config_rollbacker.checkpoint(checkpoint_name) - def delete_checkpoint(self, checkpoint_name, verbose): - config_rollbacker = self.generic_update_factory.create_config_rollbacker(verbose) - config_rollbacker.delete_checkpoint(checkpoint_name) + def delete_checkpoint(self, checkpoint_name, verbose): + config_rollbacker = self.generic_update_factory.create_config_rollbacker(verbose) + config_rollbacker.delete_checkpoint(checkpoint_name) - def list_checkpoints(self, verbose): - config_rollbacker = self.generic_update_factory.create_config_rollbacker(verbose) - return config_rollbacker.list_checkpoints() + def list_checkpoints(self, verbose): + config_rollbacker = self.generic_update_factory.create_config_rollbacker(verbose) + return config_rollbacker.list_checkpoints() From 326023c270016cf3c0c2a8669e1c6a3d006892c5 Mon Sep 17 00:00:00 2001 From: Xincun Li Date: Wed, 29 May 2024 12:06:23 -0700 Subject: [PATCH 27/44] fix format --- generic_config_updater/change_applier.py | 5 +++-- generic_config_updater/generic_updater.py | 21 +++++++++++++++++---- generic_config_updater/gu_common.py | 5 +++-- 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/generic_config_updater/change_applier.py b/generic_config_updater/change_applier.py index 74ce2353fe..0a741c26b2 100644 --- a/generic_config_updater/change_applier.py +++ b/generic_config_updater/change_applier.py @@ -168,7 +168,7 @@ def remove_backend_tables_from_config(self, data): def _get_running_config(self): _, fname = tempfile.mkstemp(suffix="_changeApplier") - + if self.scope: cmd = ['sonic-cfggen', '-d', '--print-data', '-n', self.scope] else: @@ -181,7 +181,8 @@ def _get_running_config(self): return_code = result.returncode if return_code: os.remove(fname) - raise GenericConfigUpdaterError(f"Failed to get running config for scope: {self.scope}, Return code: {return_code}, Error: {err}") + raise GenericConfigUpdaterError( + f"Failed to get running config for scope: {self.scope}, Return code: {return_code}, Error: {err}") run_data = {} try: diff --git a/generic_config_updater/generic_updater.py b/generic_config_updater/generic_updater.py index c684a63b1f..bf32a4cae0 100644 --- a/generic_config_updater/generic_updater.py +++ b/generic_config_updater/generic_updater.py @@ -120,7 +120,7 @@ def apply(self, patch, sort=True): changes_len = len(changes) self.logger.log_notice(f"The {scope} patch was converted into {changes_len} " \ - f"change{'s' if changes_len != 1 else ''}{':' if changes_len > 0 else '.'}") + f"change{'s' if changes_len != 1 else ''}{':' if changes_len > 0 else '.'}") # Apply changes in order self.logger.log_notice(f"{scope}: applying {changes_len} change{'s' if changes_len != 1 else ''} " \ @@ -367,7 +367,11 @@ def delete_checkpoint(self, name): class Decorator(PatchApplier, ConfigReplacer, FileSystemConfigRollbacker): - def __init__(self, decorated_patch_applier=None, decorated_config_replacer=None, decorated_config_rollbacker=None, scope=multi_asic.DEFAULT_NAMESPACE): + def __init__(self, + decorated_patch_applier=None, + decorated_config_replacer=None, + decorated_config_rollbacker=None, + scope=multi_asic.DEFAULT_NAMESPACE): # initing base classes to make LGTM happy PatchApplier.__init__(self, scope=scope) ConfigReplacer.__init__(self, scope=scope) @@ -396,7 +400,12 @@ def delete_checkpoint(self, checkpoint_name): class SonicYangDecorator(Decorator): - def __init__(self, patch_wrapper, config_wrapper, decorated_patch_applier=None, decorated_config_replacer=None, scope=multi_asic.DEFAULT_NAMESPACE): + def __init__(self, + patch_wrapper, + config_wrapper, + decorated_patch_applier=None, + decorated_config_replacer=None, + scope=multi_asic.DEFAULT_NAMESPACE): Decorator.__init__(self, decorated_patch_applier, decorated_config_replacer, scope=scope) self.scope = scope self.patch_wrapper = patch_wrapper @@ -418,7 +427,11 @@ def __init__(self, decorated_config_rollbacker=None, config_lock=ConfigLock(), scope=multi_asic.DEFAULT_NAMESPACE): - Decorator.__init__(self, decorated_patch_applier, decorated_config_replacer, decorated_config_rollbacker, scope=scope) + Decorator.__init__(self, + decorated_patch_applier, + decorated_config_replacer, + decorated_config_rollbacker, + scope=scope) self.config_lock = config_lock def apply(self, patch, sort=True): diff --git a/generic_config_updater/gu_common.py b/generic_config_updater/gu_common.py index 54c54c9539..d96abd680c 100644 --- a/generic_config_updater/gu_common.py +++ b/generic_config_updater/gu_common.py @@ -74,7 +74,8 @@ def _get_config_db_as_text(self): text, err = result.communicate() return_code = result.returncode if return_code: # non-zero means failure - raise GenericConfigUpdaterError(f"Failed to get running config for scope: {self.scope}, Return code: {return_code}, Error: {err}") + raise GenericConfigUpdaterError( + f"Failed to get running config for scope: {self.scope}, Return code: {return_code}, Error: {err}") return text def get_sonic_yang_as_json(self): @@ -301,7 +302,7 @@ def create_sonic_yang_with_loaded_models(self): class DryRunConfigWrapper(ConfigWrapper): # This class will simulate all read/write operations to ConfigDB on a virtual storage unit. - def __init__(self, initial_imitated_config_db = None, scope=multi_asic.DEFAULT_NAMESPACE): + def __init__(self, initial_imitated_config_db=None, scope=multi_asic.DEFAULT_NAMESPACE): super().__init__(scope=scope) self.logger = genericUpdaterLogging.get_logger(title="** DryRun", print_all_to_console=True) self.imitated_config_db = copy.deepcopy(initial_imitated_config_db) From 49ceca01760025ae619c37d7afd68490e2e729d3 Mon Sep 17 00:00:00 2001 From: Xincun Li Date: Thu, 30 May 2024 22:30:49 -0700 Subject: [PATCH 28/44] Refactor code. --- generic_config_updater/change_applier.py | 38 ++----------- generic_config_updater/generic_updater.py | 51 ++++------------- generic_config_updater/gu_common.py | 55 +++++++++++++------ tests/config_test.py | 42 +++++--------- .../change_applier_test.py | 54 ++++-------------- .../gcu_feature_patch_application_test.py | 2 +- .../multiasic_change_applier_test.py | 30 +++++----- .../multiasic_generic_updater_test.py | 6 +- 8 files changed, 95 insertions(+), 183 deletions(-) diff --git a/generic_config_updater/change_applier.py b/generic_config_updater/change_applier.py index 0a741c26b2..27a732b1f3 100644 --- a/generic_config_updater/change_applier.py +++ b/generic_config_updater/change_applier.py @@ -8,7 +8,7 @@ from collections import defaultdict from swsscommon.swsscommon import ConfigDBConnector from sonic_py_common import multi_asic -from .gu_common import GenericConfigUpdaterError, genericUpdaterLogging +from .gu_common import genericUpdaterLogging, get_config_json_by_namespace SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) UPDATER_CONF_FILE = f"{SCRIPT_DIR}/gcu_services_validator.conf.json" @@ -135,24 +135,21 @@ def _upd_data(self, tbl, run_tbl, upd_tbl, upd_keys): upd_keys[tbl][key] = {} log_debug("Patch affected tbl={} key={}".format(tbl, key)) - def _report_mismatch(self, run_data, upd_data): log_error("run_data vs expected_data: {}".format( str(jsondiff.diff(run_data, upd_data))[0:40])) - def apply(self, change): - run_data = self._get_running_config() + run_data = get_config_json_by_namespace(self.scope) upd_data = prune_empty_table(change.apply(copy.deepcopy(run_data))) upd_keys = defaultdict(dict) for tbl in sorted(set(run_data.keys()).union(set(upd_data.keys()))): - self._upd_data(tbl, run_data.get(tbl, {}), - upd_data.get(tbl, {}), upd_keys) + self._upd_data(tbl, run_data.get(tbl, {}), upd_data.get(tbl, {}), upd_keys) ret = self._services_validate(run_data, upd_data, upd_keys) if not ret: - run_data = self._get_running_config() + run_data = get_config_json_by_namespace(self.scope) self.remove_backend_tables_from_config(upd_data) self.remove_backend_tables_from_config(run_data) if upd_data != run_data: @@ -165,30 +162,3 @@ def apply(self, change): def remove_backend_tables_from_config(self, data): for key in self.backend_tables: data.pop(key, None) - - def _get_running_config(self): - _, fname = tempfile.mkstemp(suffix="_changeApplier") - - if self.scope: - cmd = ['sonic-cfggen', '-d', '--print-data', '-n', self.scope] - else: - cmd = ['sonic-cfggen', '-d', '--print-data'] - - with open(fname, "w") as file: - result = subprocess.Popen(cmd, stdout=file, stderr=subprocess.PIPE, text=True) - _, err = result.communicate() - - return_code = result.returncode - if return_code: - os.remove(fname) - raise GenericConfigUpdaterError( - f"Failed to get running config for scope: {self.scope}, Return code: {return_code}, Error: {err}") - - run_data = {} - try: - with open(fname, "r") as file: - run_data = json.load(file) - finally: - if os.path.isfile(fname): - os.remove(fname) - return run_data diff --git a/generic_config_updater/generic_updater.py b/generic_config_updater/generic_updater.py index bf32a4cae0..286ef6b349 100644 --- a/generic_config_updater/generic_updater.py +++ b/generic_config_updater/generic_updater.py @@ -5,7 +5,7 @@ from enum import Enum from .gu_common import HOST_NAMESPACE, GenericConfigUpdaterError, EmptyTableError, ConfigWrapper, \ - DryRunConfigWrapper, PatchWrapper, genericUpdaterLogging + DryRunConfigWrapper, PatchWrapper, genericUpdaterLogging, get_config_json from .patch_sorter import StrictPatchSorter, NonStrictPatchSorter, ConfigSplitter, \ TablesWithoutYangConfigSplitter, IgnorePathsFromYangConfigSplitter from .change_applier import ChangeApplier, DryRunChangeApplier @@ -36,25 +36,6 @@ def extract_scope(path): return scope, remainder -def get_cmd_output(cmd): - proc = subprocess.Popen(cmd, text=True, stdout=subprocess.PIPE) - return proc.communicate()[0], proc.returncode - - -def get_config_json_by_namespace(scope): - cmd = ['sonic-cfggen', '-d', '--print-data'] - if scope is not None and scope != multi_asic.DEFAULT_NAMESPACE: - cmd += ['-n', scope] - stdout, rc = get_cmd_output(cmd) - if rc: - raise GenericConfigUpdaterError("Failed to get cmd output '{}':rc {}".format(cmd, rc)) - try: - config_json = json.loads(stdout) - except json.JSONDecodeError as e: - raise GenericConfigUpdaterError("Failed to get config by '{}' due to {}".format(cmd, e)) - return config_json - - class ConfigLock: def acquire_lock(self): # TODO: Implement ConfigLock @@ -204,7 +185,7 @@ def checkpoint(self, checkpoint_name): self.logger.log_notice(f"Checkpoint name: {checkpoint_name}.") self.logger.log_notice("Getting current config db.") - json_content = self.config_wrapper.get_config_db_as_json() + json_content = get_config_json() self.logger.log_notice("Getting checkpoint full-path.") path = self.util.get_checkpoint_full_path(checkpoint_name) @@ -252,12 +233,10 @@ def delete_checkpoint(self, checkpoint_name): class MultiASICConfigReplacer: - def __init__(self, checkpoints_dir=CHECKPOINTS_DIR): + def __init__(self): self.logger = genericUpdaterLogging.get_logger(title="MultiASICConfigReplacer", print_all_to_console=True) - self.scopelist = [HOST_NAMESPACE].extend(multi_asic.get_namespace_list()) - self.checkpoints_dir = checkpoints_dir - self.util = Util(checkpoints_dir=checkpoints_dir) + self.scopelist = [HOST_NAMESPACE, *multi_asic.get_namespace_list()] def replace(self, target_config): config_keys = set(target_config.keys()) @@ -269,14 +248,14 @@ def replace(self, target_config): scope_config = target_config.pop(scope) if scope.lower() == HOST_NAMESPACE: scope = multi_asic.DEFAULT_NAMESPACE - ConfigReplacer(namespace=scope).replace(scope_config) + ConfigReplacer(scope=scope).replace(scope_config) class MultiASICConfigRollbacker(FileSystemConfigRollbacker): def __init__(self, checkpoints_dir=CHECKPOINTS_DIR): self.logger = genericUpdaterLogging.get_logger(title="MultiASICConfigRollbacker", print_all_to_console=True) - self.scopelist = [HOST_NAMESPACE].extend(multi_asic.get_namespace_list()) + self.scopelist = [HOST_NAMESPACE, *multi_asic.get_namespace_list()] self.checkpoints_dir = checkpoints_dir self.util = Util(checkpoints_dir=checkpoints_dir) @@ -293,26 +272,18 @@ def rollback(self, checkpoint_name): self.logger.log_notice(f"Replacing config '{checkpoint_name}' using 'Config Replacer'.") for scope in self.scopelist: - if scope.lower() == multi_asic.DEFAULT_NAMESPACE: - config = target_config.pop(HOST_NAMESPACE) - else: - config = target_config.pop(scope) - ConfigReplacer(namespace=scope).replace(config) + config = target_config.pop(scope) + if scope.lower() == HOST_NAMESPACE: + scope = multi_asic.DEFAULT_NAMESPACE + ConfigReplacer(scope=scope).replace(config) self.logger.log_notice("Config rollbacking completed.") def checkpoint(self, checkpoint_name): - all_configs = {} + all_configs = get_config_json() self.logger.log_notice("Config checkpoint starting.") self.logger.log_notice(f"Checkpoint name: {checkpoint_name}.") - for scope in self.scopelist: - self.logger.log_notice(f"Getting current {scope} config db.") - config = get_config_json_by_namespace(scope) - if scope.lower() == multi_asic.DEFAULT_NAMESPACE: - scope = HOST_NAMESPACE - all_configs[scope] = config - self.logger.log_notice("Getting checkpoint full-path.") path = self.util.get_checkpoint_full_path(checkpoint_name) diff --git a/generic_config_updater/gu_common.py b/generic_config_updater/gu_common.py index d96abd680c..95df9cb5ff 100644 --- a/generic_config_updater/gu_common.py +++ b/generic_config_updater/gu_common.py @@ -18,6 +18,42 @@ GCU_FIELD_OP_CONF_FILE = f"{SCRIPT_DIR}/gcu_field_operation_validators.conf.json" HOST_NAMESPACE = "localhost" + +def get_cmd_output(cmd): + proc = subprocess.Popen(cmd, text=True, stdout=subprocess.PIPE) + return proc.communicate()[0], proc.returncode + + +def get_config_json(): + command = ["show", "runningconfiguration", "all"] + all_running_config_text, returncode = stdout, rc = get_cmd_output(command) + if returncode: + raise GenericConfigUpdaterError( + f"Fetch all runningconfiguration failed as output:{all_running_config_text}") + all_running_config = json.loads(all_running_config_text) + + if multi_asic.is_multi_asic(): + for asic in [HOST_NAMESPACE, *multi_asic.get_namespace_list()]: + all_running_config[asic].pop("bgpraw", None) + else: + all_running_config.pop("bgpraw", None) + return all_running_config + + +def get_config_json_by_namespace(scope): + cmd = ['sonic-cfggen', '-d', '--print-data'] + if scope is not None and scope != multi_asic.DEFAULT_NAMESPACE: + cmd += ['-n', scope] + stdout, rc = get_cmd_output(cmd) + if rc: + raise GenericConfigUpdaterError("Failed to get cmd output '{}':rc {}".format(cmd, rc)) + try: + config_json = json.loads(stdout) + except json.JSONDecodeError as e: + raise GenericConfigUpdaterError("Failed to get config by '{}' due to {}".format(cmd, e)) + return config_json + + class GenericConfigUpdaterError(Exception): pass @@ -59,24 +95,7 @@ def __init__(self, yang_dir=YANG_DIR, scope=multi_asic.DEFAULT_NAMESPACE): self.sonic_yang_with_loaded_models = None def get_config_db_as_json(self): - text = self._get_config_db_as_text() - config_db_json = json.loads(text) - config_db_json.pop("bgpraw", None) - return config_db_json - - def _get_config_db_as_text(self): - if self.scope is not None and self.scope != multi_asic.DEFAULT_NAMESPACE: - cmd = ['sonic-cfggen', '-d', '--print-data', '-n', self.scope] - else: - cmd = ['sonic-cfggen', '-d', '--print-data'] - - result = subprocess.Popen(cmd, shell=False, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - text, err = result.communicate() - return_code = result.returncode - if return_code: # non-zero means failure - raise GenericConfigUpdaterError( - f"Failed to get running config for scope: {self.scope}, Return code: {return_code}, Error: {err}") - return text + return get_config_json_by_namespace(self.scope) def get_sonic_yang_as_json(self): config_db_json = self.get_config_db_as_json() diff --git a/tests/config_test.py b/tests/config_test.py index 264b5849c8..0567603e2d 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -3088,9 +3088,8 @@ def test_apply_patch_validate_patch_with_wrong_fetch_config(self, mock_subproces # Verify mocked_open was called as expected mocked_open.assert_called_with(self.patch_file_path, 'r') - @patch('config.main.SonicYangCfgDbGenerator.validate_config_db_json', mock.Mock(return_value=True)) @patch('generic_config_updater.generic_updater.ConfigReplacer.replace', MagicMock()) - def test_repalce_multiasic(self): + def test_replace_multiasic(self): # Mock open to simulate file reading mock_replace_content = copy.deepcopy(self.all_config) with patch('builtins.open', mock_open(read_data=json.dumps(mock_replace_content)), create=True) as mocked_open: @@ -3112,44 +3111,31 @@ def test_repalce_multiasic(self): # Verify mocked_open was called as expected mocked_open.assert_called_with(self.replace_file_path, 'r') - @patch('config.main.SonicYangCfgDbGenerator.validate_config_db_json', mock.Mock(return_value=True)) @patch('generic_config_updater.generic_updater.ConfigReplacer.replace', MagicMock()) - def test_repalce_multiasic_missing_scope(self): + def test_replace_multiasic_missing_scope(self): # Mock open to simulate file reading mock_replace_content = copy.deepcopy(self.all_config) mock_replace_content.pop("asic0") with patch('builtins.open', mock_open(read_data=json.dumps(mock_replace_content)), create=True) as mocked_open: - # Mock GenericUpdater to avoid actual patch application - with patch('config.main.GenericUpdater') as mock_generic_updater: - mock_generic_updater.return_value.replace = MagicMock() - - print("Multi ASIC: {}".format(multi_asic.is_multi_asic())) - # Invocation of the command with the CliRunner - result = self.runner.invoke(config.config.commands["replace"], - [self.replace_file_path], - catch_exceptions=True) - - print("Exit Code: {}, output: {}".format(result.exit_code, result.output)) - # Assertions and verifications - self.assertNotEqual(result.exit_code, 0, "Command should failed") - self.assertIn("Failed to replace config", result.output) + print("Multi ASIC: {}".format(multi_asic.is_multi_asic())) + # Invocation of the command with the CliRunner + result = self.runner.invoke(config.config.commands["replace"], + [self.replace_file_path], + catch_exceptions=True) - # Verify mocked_open was called as expected - mocked_open.assert_called_with(self.replace_file_path, 'r') + print("Exit Code: {}, output: {}".format(result.exit_code, result.output)) + # Assertions and verifications + self.assertNotEqual(result.exit_code, 0, "Command should failed") + self.assertIn("Failed to replace config", result.output) - @patch('generic_config_updater.generic_updater.subprocess.Popen') + @patch('generic_config_updater.gu_common.subprocess.Popen') @patch('generic_config_updater.generic_updater.Util.ensure_checkpoints_dir_exists', mock.Mock(return_value=True)) @patch('generic_config_updater.generic_updater.Util.save_json_file', MagicMock()) def test_checkpoint_multiasic(self, mock_subprocess_popen): allconfigs = copy.deepcopy(self.all_config) - side_effects = [ - (json.dumps(allconfigs.pop("localhost")), 0), - (json.dumps(allconfigs.pop("asic0")), 0), - (json.dumps(allconfigs.pop("asic1")), 0) - ] mock_instance = MagicMock() - mock_instance.communicate.side_effect = side_effects + mock_instance.communicate.side_effect = (allconfigs, 0) mock_instance.returncode = 0 mock_subprocess_popen.return_value = mock_instance @@ -3205,7 +3191,7 @@ def test_list_checkpoint_multiasic(self): def test_delete_checkpoint_multiasic(self): checkpointname = "checkpointname" # Mock GenericUpdater to avoid actual patch application - with patch('config.main.MultiASICConfigRollbacker') as mock_generic_updater: + with patch('config.main.GenericUpdater') as mock_generic_updater: mock_generic_updater.return_value.delete_checkpoint = MagicMock() print("Multi ASIC: {}".format(multi_asic.is_multi_asic())) diff --git a/tests/generic_config_updater/change_applier_test.py b/tests/generic_config_updater/change_applier_test.py index dbdd4992f3..ec7055641b 100644 --- a/tests/generic_config_updater/change_applier_test.py +++ b/tests/generic_config_updater/change_applier_test.py @@ -69,33 +69,16 @@ DB_HANDLE = "config_db" + def debug_print(msg): print(msg) # Mimics os.system call for sonic-cfggen -d --print-data > filename -def subprocess_Popen_cfggen(cmd, *args, **kwargs): +def mock_get_running_config_json(*args): + print(f"Get running config {args}") global running_config - - # Extract file name from kwargs if 'stdout' is a file object - stdout = kwargs.get('stdout') - if hasattr(stdout, 'name'): - fname = stdout.name - else: - raise ValueError("stdout is not a file") - - # Write the running configuration to the file specified in stdout - with open(fname, "w") as s: - json.dump(running_config, s, indent=4) - - class MockPopen: - def __init__(self): - self.returncode = 0 # Simulate successful command execution - - def communicate(self): - return "", "" # Simulate empty stdout and stderr - - return MockPopen() + return running_config # mimics config_db.set_entry @@ -189,13 +172,8 @@ def _validate_keys(keys): assert key in chg_tbl assert key in keys_tbl - -def _validate_svc(svc_name, old_cfg, new_cfg, keys): - if old_cfg != start_running_config: - debug_print("validate svc {}: old diff={}".format(svc_name, str( - jsondiff.diff(old_cfg, start_running_config)))) - assert False, "_validate_svc: old config mismatch" +def _validate_svc(svc_name, old_cfg, new_cfg, keys): if new_cfg != running_config: debug_print("validate svc {}: new diff={}".format(svc_name, str( jsondiff.diff(new_cfg, running_config)))) @@ -225,14 +203,14 @@ def vlan_validate(old_cfg, new_cfg, keys): class TestChangeApplier(unittest.TestCase): - @patch("generic_config_updater.change_applier.subprocess.Popen") + @patch("generic_config_updater.change_applier.get_config_json_by_namespace") @patch("generic_config_updater.change_applier.get_config_db") @patch("generic_config_updater.change_applier.set_config") - def test_change_apply(self, mock_set, mock_db, mock_subprocess_Popen): + def test_change_apply(self, mock_set, mock_db, mock_get_config_json): global read_data, running_config, json_changes, json_change_index global start_running_config - mock_subprocess_Popen.side_effect = subprocess_Popen_cfggen + mock_get_config_json.side_effect = mock_get_running_config_json mock_db.return_value = DB_HANDLE mock_set.side_effect = set_entry @@ -246,7 +224,7 @@ def test_change_apply(self, mock_set, mock_db, mock_subprocess_Popen): generic_config_updater.change_applier.UPDATER_CONF_FILE = CONF_FILE generic_config_updater.change_applier.set_verbose(True) generic_config_updater.services_validator.set_verbose(True) - + applier = generic_config_updater.change_applier.ChangeApplier() debug_print("invoked applier") @@ -255,7 +233,7 @@ def test_change_apply(self, mock_set, mock_db, mock_subprocess_Popen): # Take copy for comparison start_running_config = copy.deepcopy(running_config) - + debug_print("main: json_change_index={}".format(json_change_index)) applier.apply(mock_obj()) @@ -273,17 +251,6 @@ def test_change_apply(self, mock_set, mock_db, mock_subprocess_Popen): debug_print("All changes applied & tested") - # Test data is set up in such a way the multiple changes - # finally brings it back to original config. - # - if read_data["running_data"] != running_config: - debug_print("final config mismatch: {}".format(str( - jsondiff.diff(read_data["running_data"], running_config)))) - - assert read_data["running_data"] == running_config - - debug_print("all good for applier") - class TestDryRunChangeApplier(unittest.TestCase): def test_apply__calls_apply_change_to_config_db(self): @@ -298,4 +265,3 @@ def test_apply__calls_apply_change_to_config_db(self): # Assert applier.config_wrapper.apply_change_to_config_db.assert_has_calls([call(change)]) - diff --git a/tests/generic_config_updater/gcu_feature_patch_application_test.py b/tests/generic_config_updater/gcu_feature_patch_application_test.py index db625e8cd1..b9d9ece80a 100644 --- a/tests/generic_config_updater/gcu_feature_patch_application_test.py +++ b/tests/generic_config_updater/gcu_feature_patch_application_test.py @@ -87,7 +87,7 @@ def create_patch_applier(self, config): config_wrapper = self.config_wrapper config_wrapper.get_config_db_as_json = MagicMock(side_effect=get_running_config) change_applier = generic_config_updater.change_applier.ChangeApplier() - change_applier._get_running_config = MagicMock(side_effect=get_running_config) + gu.get_config_json = MagicMock(side_effect=get_running_config) patch_wrapper = PatchWrapper(config_wrapper) return gu.PatchApplier(config_wrapper=config_wrapper, patch_wrapper=patch_wrapper, changeapplier=change_applier) diff --git a/tests/generic_config_updater/multiasic_change_applier_test.py b/tests/generic_config_updater/multiasic_change_applier_test.py index e8b277618f..1af89c7192 100644 --- a/tests/generic_config_updater/multiasic_change_applier_test.py +++ b/tests/generic_config_updater/multiasic_change_applier_test.py @@ -38,9 +38,9 @@ def test_extract_scope(self): except Exception as e: assert(result == False) - @patch('generic_config_updater.change_applier.ChangeApplier._get_running_config', autospec=True) + @patch('generic_config_updater.change_applier.get_config_json_by_namespace', autospec=True) @patch('generic_config_updater.change_applier.ConfigDBConnector', autospec=True) - def test_apply_change_default_namespace(self, mock_ConfigDBConnector, mock_get_running_config): + def test_apply_change_default_scope(self, mock_ConfigDBConnector, mock_get_running_config): # Setup mock for ConfigDBConnector mock_db = MagicMock() mock_ConfigDBConnector.return_value = mock_db @@ -67,7 +67,7 @@ def test_apply_change_default_namespace(self, mock_ConfigDBConnector, mock_get_r } } - # Instantiate ChangeApplier with the default namespace + # Instantiate ChangeApplier with the default scope applier = generic_config_updater.change_applier.ChangeApplier() # Prepare a change object or data that applier.apply would use @@ -79,9 +79,9 @@ def test_apply_change_default_namespace(self, mock_ConfigDBConnector, mock_get_r # Assert ConfigDBConnector called with the correct namespace mock_ConfigDBConnector.assert_called_once_with(use_unix_socket_path=True, namespace="") - @patch('generic_config_updater.change_applier.ChangeApplier._get_running_config', autospec=True) + @patch('generic_config_updater.change_applier.get_config_json_by_namespace', autospec=True) @patch('generic_config_updater.change_applier.ConfigDBConnector', autospec=True) - def test_apply_change_given_namespace(self, mock_ConfigDBConnector, mock_get_running_config): + def test_apply_change_given_scope(self, mock_ConfigDBConnector, mock_get_running_config): # Setup mock for ConfigDBConnector mock_db = MagicMock() mock_ConfigDBConnector.return_value = mock_db @@ -108,8 +108,8 @@ def test_apply_change_given_namespace(self, mock_ConfigDBConnector, mock_get_run } } - # Instantiate ChangeApplier with the default namespace - applier = generic_config_updater.change_applier.ChangeApplier(namespace="asic0") + # Instantiate ChangeApplier with the default scope + applier = generic_config_updater.change_applier.ChangeApplier(scope="asic0") # Prepare a change object or data that applier.apply would use change = MagicMock() @@ -117,10 +117,10 @@ def test_apply_change_given_namespace(self, mock_ConfigDBConnector, mock_get_run # Call the apply method with the change object applier.apply(change) - # Assert ConfigDBConnector called with the correct namespace + # Assert ConfigDBConnector called with the correct scope mock_ConfigDBConnector.assert_called_once_with(use_unix_socket_path=True, namespace="asic0") - @patch('generic_config_updater.change_applier.ChangeApplier._get_running_config', autospec=True) + @patch('generic_config_updater.change_applier.get_config_json_by_namespace', autospec=True) @patch('generic_config_updater.change_applier.ConfigDBConnector', autospec=True) def test_apply_change_failure(self, mock_ConfigDBConnector, mock_get_running_config): # Setup mock for ConfigDBConnector @@ -129,9 +129,9 @@ def test_apply_change_failure(self, mock_ConfigDBConnector, mock_get_running_con # Setup mock for json.load to return some running configuration mock_get_running_config.side_effect = Exception("Failed to get running config") - # Instantiate ChangeApplier with a specific namespace to simulate applying changes in a multi-asic environment - namespace = "asic0" - applier = generic_config_updater.change_applier.ChangeApplier(namespace=namespace) + # Instantiate ChangeApplier with a specific scope to simulate applying changes in a multi-asic environment + scope = "asic0" + applier = generic_config_updater.change_applier.ChangeApplier(scope=scope) # Prepare a change object or data that applier.apply would use change = MagicMock() @@ -142,7 +142,7 @@ def test_apply_change_failure(self, mock_ConfigDBConnector, mock_get_running_con self.assertTrue('Failed to get running config' in str(context.exception)) - @patch('generic_config_updater.change_applier.ChangeApplier._get_running_config', autospec=True) + @patch('generic_config_updater.change_applier.get_config_json_by_namespace', autospec=True) @patch('generic_config_updater.change_applier.ConfigDBConnector', autospec=True) def test_apply_patch_with_empty_tables_failure(self, mock_ConfigDBConnector, mock_get_running_config): # Setup mock for ConfigDBConnector @@ -159,8 +159,8 @@ def test_apply_patch_with_empty_tables_failure(self, mock_ConfigDBConnector, moc } } - # Instantiate ChangeApplier with a specific namespace to simulate applying changes in a multi-asic environment - applier = generic_config_updater.change_applier.ChangeApplier(namespace="asic0") + # Instantiate ChangeApplier with a specific scope to simulate applying changes in a multi-asic environment + applier = generic_config_updater.change_applier.ChangeApplier(scope="asic0") # Prepare a change object or data that applier.apply would use, simulating a patch that requires non-empty tables change = MagicMock() diff --git a/tests/generic_config_updater/multiasic_generic_updater_test.py b/tests/generic_config_updater/multiasic_generic_updater_test.py index 4a55eb98be..5acdd391f0 100644 --- a/tests/generic_config_updater/multiasic_generic_updater_test.py +++ b/tests/generic_config_updater/multiasic_generic_updater_test.py @@ -19,7 +19,7 @@ class TestMultiAsicPatchApplier(unittest.TestCase): @patch('generic_config_updater.gu_common.PatchWrapper.simulate_patch') @patch('generic_config_updater.generic_updater.ChangeApplier') def test_apply_patch_specific_namespace(self, mock_ChangeApplier, mock_simulate_patch, mock_get_config, mock_get_empty_tables): - namespace = "asic0" + scope = "asic0" patch_data = jsonpatch.JsonPatch([ { "op": "add", @@ -158,10 +158,10 @@ def test_apply_patch_specific_namespace(self, mock_ChangeApplier, mock_simulate_ } } - patch_applier = generic_config_updater.generic_updater.PatchApplier(namespace=namespace) + patch_applier = generic_config_updater.generic_updater.PatchApplier(scope=scope) # Apply the patch and verify patch_applier.apply(patch_data) # Assertions to ensure the namespace is correctly used in underlying calls - mock_ChangeApplier.assert_called_once_with(namespace=namespace) + mock_ChangeApplier.assert_called_once_with(scope=scope) From 33d6801a28e10ee2bb9588a7af47e5bf13dc05d1 Mon Sep 17 00:00:00 2001 From: Xincun Li Date: Thu, 30 May 2024 22:35:28 -0700 Subject: [PATCH 29/44] remove unused part. --- generic_config_updater/generic_updater.py | 1 - tests/config_test.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/generic_config_updater/generic_updater.py b/generic_config_updater/generic_updater.py index 286ef6b349..cfe7042cae 100644 --- a/generic_config_updater/generic_updater.py +++ b/generic_config_updater/generic_updater.py @@ -1,7 +1,6 @@ import json import jsonpointer import os -import subprocess from enum import Enum from .gu_common import HOST_NAMESPACE, GenericConfigUpdaterError, EmptyTableError, ConfigWrapper, \ diff --git a/tests/config_test.py b/tests/config_test.py index 0567603e2d..b25807bf5a 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -3116,7 +3116,7 @@ def test_replace_multiasic_missing_scope(self): # Mock open to simulate file reading mock_replace_content = copy.deepcopy(self.all_config) mock_replace_content.pop("asic0") - with patch('builtins.open', mock_open(read_data=json.dumps(mock_replace_content)), create=True) as mocked_open: + with patch('builtins.open', mock_open(read_data=json.dumps(mock_replace_content)), create=True): print("Multi ASIC: {}".format(multi_asic.is_multi_asic())) # Invocation of the command with the CliRunner result = self.runner.invoke(config.config.commands["replace"], From ac850323e699048620107df265127e2681502e6d Mon Sep 17 00:00:00 2001 From: Xincun Li Date: Fri, 31 May 2024 10:06:49 -0700 Subject: [PATCH 30/44] fix show import --- show/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/show/main.py b/show/main.py index c970cd667f..15eca2e4b9 100755 --- a/show/main.py +++ b/show/main.py @@ -5,7 +5,7 @@ import re import click -from generic_config_updater.generic_updater import get_config_json_by_namespace +from generic_config_updater.gu_common import get_config_json_by_namespace import lazy_object_proxy import utilities_common.cli as clicommon from sonic_py_common import multi_asic From d0116e14e4aa8380a76fc41cf6242e44433cd227 Mon Sep 17 00:00:00 2001 From: Xincun Li Date: Fri, 31 May 2024 14:57:35 -0700 Subject: [PATCH 31/44] fix ut. --- generic_config_updater/change_applier.py | 34 ++++++++++-- .../change_applier_test.py | 54 +++++++++++++++---- .../gcu_feature_patch_application_test.py | 2 +- .../multiasic_change_applier_test.py | 8 +-- tests/show_test.py | 9 ++-- 5 files changed, 84 insertions(+), 23 deletions(-) diff --git a/generic_config_updater/change_applier.py b/generic_config_updater/change_applier.py index 27a732b1f3..ab1ca8bfd2 100644 --- a/generic_config_updater/change_applier.py +++ b/generic_config_updater/change_applier.py @@ -8,7 +8,7 @@ from collections import defaultdict from swsscommon.swsscommon import ConfigDBConnector from sonic_py_common import multi_asic -from .gu_common import genericUpdaterLogging, get_config_json_by_namespace +from .gu_common import GenericConfigUpdaterError, genericUpdaterLogging SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) UPDATER_CONF_FILE = f"{SCRIPT_DIR}/gcu_services_validator.conf.json" @@ -140,7 +140,7 @@ def _report_mismatch(self, run_data, upd_data): str(jsondiff.diff(run_data, upd_data))[0:40])) def apply(self, change): - run_data = get_config_json_by_namespace(self.scope) + run_data = self._get_running_config(self.scope) upd_data = prune_empty_table(change.apply(copy.deepcopy(run_data))) upd_keys = defaultdict(dict) @@ -149,7 +149,7 @@ def apply(self, change): ret = self._services_validate(run_data, upd_data, upd_keys) if not ret: - run_data = get_config_json_by_namespace(self.scope) + run_data = self._get_running_config(self.scope) self.remove_backend_tables_from_config(upd_data) self.remove_backend_tables_from_config(run_data) if upd_data != run_data: @@ -162,3 +162,31 @@ def apply(self, change): def remove_backend_tables_from_config(self, data): for key in self.backend_tables: data.pop(key, None) + + def _get_running_config(self): + _, fname = tempfile.mkstemp(suffix="_changeApplier") + + if self.namespace: + cmd = ['sonic-cfggen', '-d', '--print-data', '-n', self.namespace] + else: + cmd = ['sonic-cfggen', '-d', '--print-data'] + + with open(fname, "w") as file: + result = subprocess.Popen(cmd, stdout=file, stderr=subprocess.PIPE, text=True) + _, err = result.communicate() + + return_code = result.returncode + if return_code: + os.remove(fname) + raise GenericConfigUpdaterError( + f"Failed to get running config for namespace: {self.scope}," + + f"Return code: {return_code}, Error: {err}") + + run_data = {} + try: + with open(fname, "r") as file: + run_data = json.load(file) + finally: + if os.path.isfile(fname): + os.remove(fname) + return run_data \ No newline at end of file diff --git a/tests/generic_config_updater/change_applier_test.py b/tests/generic_config_updater/change_applier_test.py index ec7055641b..7a3f1c2662 100644 --- a/tests/generic_config_updater/change_applier_test.py +++ b/tests/generic_config_updater/change_applier_test.py @@ -69,16 +69,33 @@ DB_HANDLE = "config_db" - def debug_print(msg): print(msg) # Mimics os.system call for sonic-cfggen -d --print-data > filename -def mock_get_running_config_json(*args): - print(f"Get running config {args}") +def subprocess_Popen_cfggen(cmd, *args, **kwargs): global running_config - return running_config + + # Extract file name from kwargs if 'stdout' is a file object + stdout = kwargs.get('stdout') + if hasattr(stdout, 'name'): + fname = stdout.name + else: + raise ValueError("stdout is not a file") + + # Write the running configuration to the file specified in stdout + with open(fname, "w") as s: + json.dump(running_config, s, indent=4) + + class MockPopen: + def __init__(self): + self.returncode = 0 # Simulate successful command execution + + def communicate(self): + return "", "" # Simulate empty stdout and stderr + + return MockPopen() # mimics config_db.set_entry @@ -172,8 +189,13 @@ def _validate_keys(keys): assert key in chg_tbl assert key in keys_tbl - + def _validate_svc(svc_name, old_cfg, new_cfg, keys): + if old_cfg != start_running_config: + debug_print("validate svc {}: old diff={}".format(svc_name, str( + jsondiff.diff(old_cfg, start_running_config)))) + assert False, "_validate_svc: old config mismatch" + if new_cfg != running_config: debug_print("validate svc {}: new diff={}".format(svc_name, str( jsondiff.diff(new_cfg, running_config)))) @@ -203,14 +225,14 @@ def vlan_validate(old_cfg, new_cfg, keys): class TestChangeApplier(unittest.TestCase): - @patch("generic_config_updater.change_applier.get_config_json_by_namespace") + @patch("generic_config_updater.change_applier.subprocess.Popen") @patch("generic_config_updater.change_applier.get_config_db") @patch("generic_config_updater.change_applier.set_config") - def test_change_apply(self, mock_set, mock_db, mock_get_config_json): + def test_change_apply(self, mock_set, mock_db, mock_subprocess_Popen): global read_data, running_config, json_changes, json_change_index global start_running_config - mock_get_config_json.side_effect = mock_get_running_config_json + mock_subprocess_Popen.side_effect = subprocess_Popen_cfggen mock_db.return_value = DB_HANDLE mock_set.side_effect = set_entry @@ -220,11 +242,10 @@ def test_change_apply(self, mock_set, mock_db, mock_get_config_json): running_config = copy.deepcopy(read_data["running_data"]) json_changes = copy.deepcopy(read_data["json_changes"]) - generic_config_updater.change_applier.ChangeApplier.updater_conf = None generic_config_updater.change_applier.UPDATER_CONF_FILE = CONF_FILE generic_config_updater.change_applier.set_verbose(True) generic_config_updater.services_validator.set_verbose(True) - + applier = generic_config_updater.change_applier.ChangeApplier() debug_print("invoked applier") @@ -233,7 +254,7 @@ def test_change_apply(self, mock_set, mock_db, mock_get_config_json): # Take copy for comparison start_running_config = copy.deepcopy(running_config) - + debug_print("main: json_change_index={}".format(json_change_index)) applier.apply(mock_obj()) @@ -251,6 +272,17 @@ def test_change_apply(self, mock_set, mock_db, mock_get_config_json): debug_print("All changes applied & tested") + # Test data is set up in such a way the multiple changes + # finally brings it back to original config. + # + if read_data["running_data"] != running_config: + debug_print("final config mismatch: {}".format(str( + jsondiff.diff(read_data["running_data"], running_config)))) + + assert read_data["running_data"] == running_config + + debug_print("all good for applier") + class TestDryRunChangeApplier(unittest.TestCase): def test_apply__calls_apply_change_to_config_db(self): diff --git a/tests/generic_config_updater/gcu_feature_patch_application_test.py b/tests/generic_config_updater/gcu_feature_patch_application_test.py index b9d9ece80a..db625e8cd1 100644 --- a/tests/generic_config_updater/gcu_feature_patch_application_test.py +++ b/tests/generic_config_updater/gcu_feature_patch_application_test.py @@ -87,7 +87,7 @@ def create_patch_applier(self, config): config_wrapper = self.config_wrapper config_wrapper.get_config_db_as_json = MagicMock(side_effect=get_running_config) change_applier = generic_config_updater.change_applier.ChangeApplier() - gu.get_config_json = MagicMock(side_effect=get_running_config) + change_applier._get_running_config = MagicMock(side_effect=get_running_config) patch_wrapper = PatchWrapper(config_wrapper) return gu.PatchApplier(config_wrapper=config_wrapper, patch_wrapper=patch_wrapper, changeapplier=change_applier) diff --git a/tests/generic_config_updater/multiasic_change_applier_test.py b/tests/generic_config_updater/multiasic_change_applier_test.py index 1af89c7192..cbeb0ef7b9 100644 --- a/tests/generic_config_updater/multiasic_change_applier_test.py +++ b/tests/generic_config_updater/multiasic_change_applier_test.py @@ -38,7 +38,7 @@ def test_extract_scope(self): except Exception as e: assert(result == False) - @patch('generic_config_updater.change_applier.get_config_json_by_namespace', autospec=True) + @patch('generic_config_updater.change_applier._get_running_config', autospec=True) @patch('generic_config_updater.change_applier.ConfigDBConnector', autospec=True) def test_apply_change_default_scope(self, mock_ConfigDBConnector, mock_get_running_config): # Setup mock for ConfigDBConnector @@ -79,7 +79,7 @@ def test_apply_change_default_scope(self, mock_ConfigDBConnector, mock_get_runni # Assert ConfigDBConnector called with the correct namespace mock_ConfigDBConnector.assert_called_once_with(use_unix_socket_path=True, namespace="") - @patch('generic_config_updater.change_applier.get_config_json_by_namespace', autospec=True) + @patch('generic_config_updater.change_applier._get_running_config', autospec=True) @patch('generic_config_updater.change_applier.ConfigDBConnector', autospec=True) def test_apply_change_given_scope(self, mock_ConfigDBConnector, mock_get_running_config): # Setup mock for ConfigDBConnector @@ -120,7 +120,7 @@ def test_apply_change_given_scope(self, mock_ConfigDBConnector, mock_get_running # Assert ConfigDBConnector called with the correct scope mock_ConfigDBConnector.assert_called_once_with(use_unix_socket_path=True, namespace="asic0") - @patch('generic_config_updater.change_applier.get_config_json_by_namespace', autospec=True) + @patch('generic_config_updater.change_applier._get_running_config', autospec=True) @patch('generic_config_updater.change_applier.ConfigDBConnector', autospec=True) def test_apply_change_failure(self, mock_ConfigDBConnector, mock_get_running_config): # Setup mock for ConfigDBConnector @@ -142,7 +142,7 @@ def test_apply_change_failure(self, mock_ConfigDBConnector, mock_get_running_con self.assertTrue('Failed to get running config' in str(context.exception)) - @patch('generic_config_updater.change_applier.get_config_json_by_namespace', autospec=True) + @patch('generic_config_updater.change_applier._get_running_config', autospec=True) @patch('generic_config_updater.change_applier.ConfigDBConnector', autospec=True) def test_apply_patch_with_empty_tables_failure(self, mock_ConfigDBConnector, mock_get_running_config): # Setup mock for ConfigDBConnector diff --git a/tests/show_test.py b/tests/show_test.py index ce9e82c531..30333d6ff8 100644 --- a/tests/show_test.py +++ b/tests/show_test.py @@ -41,7 +41,7 @@ def setup_class(cls): def mock_run_bgp_command(): return "" - @patch('generic_config_updater.generic_updater.subprocess.Popen') + @patch('generic_config_updater.gu_common.subprocess.Popen') def test_show_runningconfiguration_all_json_loads_failure(self, mock_subprocess_popen): mock_instance = MagicMock() mock_instance.communicate.return_value = ("", 2) @@ -49,7 +49,7 @@ def test_show_runningconfiguration_all_json_loads_failure(self, mock_subprocess_ result = CliRunner().invoke(show.cli.commands['runningconfiguration'].commands['all'], []) assert result.exit_code != 0 - @patch('generic_config_updater.generic_updater.subprocess.Popen') + @patch('generic_config_updater.gu_common.subprocess.Popen') def test_show_runningconfiguration_all_get_cmd_ouput_failure(self, mock_subprocess_popen): mock_instance = MagicMock() mock_instance.communicate.return_value = ("{}", 2) @@ -57,7 +57,7 @@ def test_show_runningconfiguration_all_get_cmd_ouput_failure(self, mock_subproce result = CliRunner().invoke(show.cli.commands['runningconfiguration'].commands['all'], []) assert result.exit_code != 0 - @patch('generic_config_updater.generic_updater.subprocess.Popen') + @patch('generic_config_updater.gu_common.subprocess.Popen') def test_show_runningconfiguration_all(self, mock_subprocess_popen): mock_instance = MagicMock() mock_instance.communicate.return_value = ("{}", 0) @@ -98,9 +98,10 @@ def mock_run_bgp_command(): def test_show_runningconfiguration_all_masic(self): def get_cmd_output_side_effect(*args, **kwargs): return "{}", 0 - with mock.patch('generic_config_updater.generic_updater.get_cmd_output', + with mock.patch('generic_config_updater.gu_common.get_cmd_output', mock.MagicMock(side_effect=get_cmd_output_side_effect)) as mock_get_cmd_output: result = CliRunner().invoke(show.cli.commands['runningconfiguration'].commands['all'], []) + print(result) assert result.exit_code == 0 assert mock_get_cmd_output.call_count == 3 assert mock_get_cmd_output.call_args_list == [ From 26c84cc013e45ef16a2911a97bbc67d4f26f3fb2 Mon Sep 17 00:00:00 2001 From: Xincun Li Date: Fri, 31 May 2024 15:00:22 -0700 Subject: [PATCH 32/44] fix format --- generic_config_updater/change_applier.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generic_config_updater/change_applier.py b/generic_config_updater/change_applier.py index ab1ca8bfd2..6bc97bd105 100644 --- a/generic_config_updater/change_applier.py +++ b/generic_config_updater/change_applier.py @@ -189,4 +189,4 @@ def _get_running_config(self): finally: if os.path.isfile(fname): os.remove(fname) - return run_data \ No newline at end of file + return run_data From 4465fdcb5b21e8da065202338d3b628f40d8e93e Mon Sep 17 00:00:00 2001 From: Xincun Li Date: Fri, 31 May 2024 15:31:47 -0700 Subject: [PATCH 33/44] fix ut. --- generic_config_updater/change_applier.py | 10 +++--- show/main.py | 23 +++++++++++- tests/show_test.py | 46 ++++++++++++------------ 3 files changed, 49 insertions(+), 30 deletions(-) diff --git a/generic_config_updater/change_applier.py b/generic_config_updater/change_applier.py index 6bc97bd105..f41835945b 100644 --- a/generic_config_updater/change_applier.py +++ b/generic_config_updater/change_applier.py @@ -140,7 +140,7 @@ def _report_mismatch(self, run_data, upd_data): str(jsondiff.diff(run_data, upd_data))[0:40])) def apply(self, change): - run_data = self._get_running_config(self.scope) + run_data = self._get_running_config() upd_data = prune_empty_table(change.apply(copy.deepcopy(run_data))) upd_keys = defaultdict(dict) @@ -149,7 +149,7 @@ def apply(self, change): ret = self._services_validate(run_data, upd_data, upd_keys) if not ret: - run_data = self._get_running_config(self.scope) + run_data = self._get_running_config() self.remove_backend_tables_from_config(upd_data) self.remove_backend_tables_from_config(run_data) if upd_data != run_data: @@ -166,8 +166,8 @@ def remove_backend_tables_from_config(self, data): def _get_running_config(self): _, fname = tempfile.mkstemp(suffix="_changeApplier") - if self.namespace: - cmd = ['sonic-cfggen', '-d', '--print-data', '-n', self.namespace] + if self.scope: + cmd = ['sonic-cfggen', '-d', '--print-data', '-n', self.scope] else: cmd = ['sonic-cfggen', '-d', '--print-data'] @@ -179,7 +179,7 @@ def _get_running_config(self): if return_code: os.remove(fname) raise GenericConfigUpdaterError( - f"Failed to get running config for namespace: {self.scope}," + + f"Failed to get running config for scope: {self.scope}," + f"Return code: {return_code}, Error: {err}") run_data = {} diff --git a/show/main.py b/show/main.py index 15eca2e4b9..cfdf30d3c6 100755 --- a/show/main.py +++ b/show/main.py @@ -5,7 +5,6 @@ import re import click -from generic_config_updater.gu_common import get_config_json_by_namespace import lazy_object_proxy import utilities_common.cli as clicommon from sonic_py_common import multi_asic @@ -139,6 +138,28 @@ def run_command(command, display_cmd=False, return_cmd=False, shell=False): if rc != 0: sys.exit(rc) +def get_cmd_output(cmd): + proc = subprocess.Popen(cmd, text=True, stdout=subprocess.PIPE) + return proc.communicate()[0], proc.returncode + +def get_config_json_by_namespace(namespace): + cmd = ['sonic-cfggen', '-d', '--print-data'] + if namespace is not None and namespace != multi_asic.DEFAULT_NAMESPACE: + cmd += ['-n', namespace] + + stdout, rc = get_cmd_output(cmd) + if rc: + click.echo("Failed to get cmd output '{}':rc {}".format(cmd, rc)) + raise click.Abort() + + try: + config_json = json.loads(stdout) + except JSONDecodeError as e: + click.echo("Failed to load output '{}':{}".format(cmd, e)) + raise click.Abort() + + return config_json + # Lazy global class instance for SONiC interface name to alias conversion iface_alias_converter = lazy_object_proxy.Proxy(lambda: clicommon.InterfaceAliasConverter()) diff --git a/tests/show_test.py b/tests/show_test.py index 30333d6ff8..4cd29ac45e 100644 --- a/tests/show_test.py +++ b/tests/show_test.py @@ -41,33 +41,32 @@ def setup_class(cls): def mock_run_bgp_command(): return "" - @patch('generic_config_updater.gu_common.subprocess.Popen') - def test_show_runningconfiguration_all_json_loads_failure(self, mock_subprocess_popen): - mock_instance = MagicMock() - mock_instance.communicate.return_value = ("", 2) - mock_subprocess_popen.return_value = mock_instance - result = CliRunner().invoke(show.cli.commands['runningconfiguration'].commands['all'], []) + def test_show_runningconfiguration_all_json_loads_failure(self): + def get_cmd_output_side_effect(*args, **kwargs): + return "", 0 + with mock.patch('show.main.get_cmd_output', + mock.MagicMock(side_effect=get_cmd_output_side_effect)) as mock_get_cmd_output: + result = CliRunner().invoke(show.cli.commands['runningconfiguration'].commands['all'], []) assert result.exit_code != 0 - @patch('generic_config_updater.gu_common.subprocess.Popen') - def test_show_runningconfiguration_all_get_cmd_ouput_failure(self, mock_subprocess_popen): - mock_instance = MagicMock() - mock_instance.communicate.return_value = ("{}", 2) - mock_subprocess_popen.return_value = mock_instance - result = CliRunner().invoke(show.cli.commands['runningconfiguration'].commands['all'], []) + def test_show_runningconfiguration_all_get_cmd_ouput_failure(self): + def get_cmd_output_side_effect(*args, **kwargs): + return "{}", 2 + with mock.patch('show.main.get_cmd_output', + mock.MagicMock(side_effect=get_cmd_output_side_effect)) as mock_get_cmd_output: + result = CliRunner().invoke(show.cli.commands['runningconfiguration'].commands['all'], []) assert result.exit_code != 0 - @patch('generic_config_updater.gu_common.subprocess.Popen') - def test_show_runningconfiguration_all(self, mock_subprocess_popen): - mock_instance = MagicMock() - mock_instance.communicate.return_value = ("{}", 0) - mock_subprocess_popen.return_value = mock_instance - mock_instance.returncode = 0 - result = CliRunner().invoke(show.cli.commands['runningconfiguration'].commands['all'], []) + def test_show_runningconfiguration_all(self): + def get_cmd_output_side_effect(*args, **kwargs): + return "{}", 0 + with mock.patch('show.main.get_cmd_output', + mock.MagicMock(side_effect=get_cmd_output_side_effect)) as mock_get_cmd_output: + result = CliRunner().invoke(show.cli.commands['runningconfiguration'].commands['all'], []) assert result.exit_code == 0 - assert mock_subprocess_popen.call_count == 1 - assert mock_subprocess_popen.call_args_list == [ - call(['sonic-cfggen', '-d', '--print-data'], text=True, stdout=-1)] + assert mock_get_cmd_output.call_count == 1 + assert mock_get_cmd_output.call_args_list == [ + call(['sonic-cfggen', '-d', '--print-data'])] @classmethod def teardown_class(cls): @@ -98,10 +97,9 @@ def mock_run_bgp_command(): def test_show_runningconfiguration_all_masic(self): def get_cmd_output_side_effect(*args, **kwargs): return "{}", 0 - with mock.patch('generic_config_updater.gu_common.get_cmd_output', + with mock.patch('show.main.get_cmd_output', mock.MagicMock(side_effect=get_cmd_output_side_effect)) as mock_get_cmd_output: result = CliRunner().invoke(show.cli.commands['runningconfiguration'].commands['all'], []) - print(result) assert result.exit_code == 0 assert mock_get_cmd_output.call_count == 3 assert mock_get_cmd_output.call_args_list == [ From 906e903de3ef631bea9b4981a17e5d51c3d17d1f Mon Sep 17 00:00:00 2001 From: Xincun Li Date: Mon, 3 Jun 2024 10:15:07 -0700 Subject: [PATCH 34/44] fix ut --- generic_config_updater/gu_common.py | 18 +++++++++++++++++- .../change_applier_test.py | 5 +++-- .../generic_updater_test.py | 4 +++- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/generic_config_updater/gu_common.py b/generic_config_updater/gu_common.py index 95df9cb5ff..c91913f658 100644 --- a/generic_config_updater/gu_common.py +++ b/generic_config_updater/gu_common.py @@ -95,7 +95,23 @@ def __init__(self, yang_dir=YANG_DIR, scope=multi_asic.DEFAULT_NAMESPACE): self.sonic_yang_with_loaded_models = None def get_config_db_as_json(self): - return get_config_json_by_namespace(self.scope) + text = self._get_config_db_as_text() + config_db_json = json.loads(text) + config_db_json.pop("bgpraw", None) + return config_db_json + + def _get_config_db_as_text(self): + if self.namespace is not None and self.namespace != multi_asic.DEFAULT_NAMESPACE: + cmd = ['sonic-cfggen', '-d', '--print-data', '-n', self.namespace] + else: + cmd = ['sonic-cfggen', '-d', '--print-data'] + + result = subprocess.Popen(cmd, shell=False, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + text, err = result.communicate() + return_code = result.returncode + if return_code: # non-zero means failure + raise GenericConfigUpdaterError(f"Failed to get running config for namespace: {self.namespace}, Return code: {return_code}, Error: {err}") + return text def get_sonic_yang_as_json(self): config_db_json = self.get_config_db_as_json() diff --git a/tests/generic_config_updater/change_applier_test.py b/tests/generic_config_updater/change_applier_test.py index 7a3f1c2662..7aad111f18 100644 --- a/tests/generic_config_updater/change_applier_test.py +++ b/tests/generic_config_updater/change_applier_test.py @@ -242,10 +242,11 @@ def test_change_apply(self, mock_set, mock_db, mock_subprocess_Popen): running_config = copy.deepcopy(read_data["running_data"]) json_changes = copy.deepcopy(read_data["json_changes"]) + generic_config_updater.change_applier.ChangeApplier.updater_conf = None generic_config_updater.change_applier.UPDATER_CONF_FILE = CONF_FILE generic_config_updater.change_applier.set_verbose(True) generic_config_updater.services_validator.set_verbose(True) - + applier = generic_config_updater.change_applier.ChangeApplier() debug_print("invoked applier") @@ -254,7 +255,7 @@ def test_change_apply(self, mock_set, mock_db, mock_subprocess_Popen): # Take copy for comparison start_running_config = copy.deepcopy(running_config) - + debug_print("main: json_change_index={}".format(json_change_index)) applier.apply(mock_obj()) diff --git a/tests/generic_config_updater/generic_updater_test.py b/tests/generic_config_updater/generic_updater_test.py index 96c25e3552..a62b7d38d5 100644 --- a/tests/generic_config_updater/generic_updater_test.py +++ b/tests/generic_config_updater/generic_updater_test.py @@ -2,7 +2,7 @@ import os import shutil import unittest -from unittest.mock import MagicMock, Mock, call +from unittest.mock import MagicMock, Mock, call, patch from .gutest_helpers import create_side_effect_dict, Files import generic_config_updater.generic_updater as gu @@ -124,6 +124,8 @@ def __create_config_replacer(self, changes=None, verified_same_config=True): return gu.ConfigReplacer(patch_applier, config_wrapper, patch_wrapper) + +@patch('generic_config_updater.gu_common.get_config_json', MagicMock(return_value={})) class TestFileSystemConfigRollbacker(unittest.TestCase): def setUp(self): self.checkpoints_dir = os.path.join(os.getcwd(),"checkpoints") From 698959b1241411db1ad59407b536732220d76a6e Mon Sep 17 00:00:00 2001 From: Xincun Li Date: Mon, 3 Jun 2024 10:57:53 -0700 Subject: [PATCH 35/44] fix ut --- tests/generic_config_updater/generic_updater_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/generic_config_updater/generic_updater_test.py b/tests/generic_config_updater/generic_updater_test.py index a62b7d38d5..8480dc23b0 100644 --- a/tests/generic_config_updater/generic_updater_test.py +++ b/tests/generic_config_updater/generic_updater_test.py @@ -125,7 +125,7 @@ def __create_config_replacer(self, changes=None, verified_same_config=True): return gu.ConfigReplacer(patch_applier, config_wrapper, patch_wrapper) -@patch('generic_config_updater.gu_common.get_config_json', MagicMock(return_value={})) +@patch('generic_config_updater.generic_updater.get_config_json', MagicMock(return_value={})) class TestFileSystemConfigRollbacker(unittest.TestCase): def setUp(self): self.checkpoints_dir = os.path.join(os.getcwd(),"checkpoints") From 5f2c6a6b020451eb1e467d67197fac8840e278f0 Mon Sep 17 00:00:00 2001 From: Xincun Li Date: Mon, 3 Jun 2024 14:02:28 -0700 Subject: [PATCH 36/44] fix ut. --- generic_config_updater/generic_updater.py | 28 +++++++++++++-- generic_config_updater/gu_common.py | 35 ------------------- .../multiasic_change_applier_test.py | 8 ++--- 3 files changed, 29 insertions(+), 42 deletions(-) diff --git a/generic_config_updater/generic_updater.py b/generic_config_updater/generic_updater.py index cfe7042cae..093f0ee655 100644 --- a/generic_config_updater/generic_updater.py +++ b/generic_config_updater/generic_updater.py @@ -1,10 +1,11 @@ import json +import subprocess import jsonpointer import os from enum import Enum from .gu_common import HOST_NAMESPACE, GenericConfigUpdaterError, EmptyTableError, ConfigWrapper, \ - DryRunConfigWrapper, PatchWrapper, genericUpdaterLogging, get_config_json + DryRunConfigWrapper, PatchWrapper, genericUpdaterLogging from .patch_sorter import StrictPatchSorter, NonStrictPatchSorter, ConfigSplitter, \ TablesWithoutYangConfigSplitter, IgnorePathsFromYangConfigSplitter from .change_applier import ChangeApplier, DryRunChangeApplier @@ -35,6 +36,27 @@ def extract_scope(path): return scope, remainder +def get_cmd_output(cmd): + proc = subprocess.Popen(cmd, text=True, stdout=subprocess.PIPE) + return proc.communicate()[0], proc.returncode + + +def get_config_json(): + command = ["show", "runningconfiguration", "all"] + all_running_config_text, returncode = get_cmd_output(command) + if returncode: + raise GenericConfigUpdaterError( + f"Fetch all runningconfiguration failed as output:{all_running_config_text}") + all_running_config = json.loads(all_running_config_text) + + if multi_asic.is_multi_asic(): + for asic in [HOST_NAMESPACE, *multi_asic.get_namespace_list()]: + all_running_config[asic].pop("bgpraw", None) + else: + all_running_config.pop("bgpraw", None) + return all_running_config + + class ConfigLock: def acquire_lock(self): # TODO: Implement ConfigLock @@ -231,7 +253,7 @@ def delete_checkpoint(self, checkpoint_name): self.logger.log_notice("Deleting checkpoint completed.") -class MultiASICConfigReplacer: +class MultiASICConfigReplacer(ConfigReplacer): def __init__(self): self.logger = genericUpdaterLogging.get_logger(title="MultiASICConfigReplacer", print_all_to_console=True) @@ -247,7 +269,7 @@ def replace(self, target_config): scope_config = target_config.pop(scope) if scope.lower() == HOST_NAMESPACE: scope = multi_asic.DEFAULT_NAMESPACE - ConfigReplacer(scope=scope).replace(scope_config) + super(scope=scope).replace(scope_config) class MultiASICConfigRollbacker(FileSystemConfigRollbacker): diff --git a/generic_config_updater/gu_common.py b/generic_config_updater/gu_common.py index c91913f658..e68b4b5131 100644 --- a/generic_config_updater/gu_common.py +++ b/generic_config_updater/gu_common.py @@ -19,41 +19,6 @@ HOST_NAMESPACE = "localhost" -def get_cmd_output(cmd): - proc = subprocess.Popen(cmd, text=True, stdout=subprocess.PIPE) - return proc.communicate()[0], proc.returncode - - -def get_config_json(): - command = ["show", "runningconfiguration", "all"] - all_running_config_text, returncode = stdout, rc = get_cmd_output(command) - if returncode: - raise GenericConfigUpdaterError( - f"Fetch all runningconfiguration failed as output:{all_running_config_text}") - all_running_config = json.loads(all_running_config_text) - - if multi_asic.is_multi_asic(): - for asic in [HOST_NAMESPACE, *multi_asic.get_namespace_list()]: - all_running_config[asic].pop("bgpraw", None) - else: - all_running_config.pop("bgpraw", None) - return all_running_config - - -def get_config_json_by_namespace(scope): - cmd = ['sonic-cfggen', '-d', '--print-data'] - if scope is not None and scope != multi_asic.DEFAULT_NAMESPACE: - cmd += ['-n', scope] - stdout, rc = get_cmd_output(cmd) - if rc: - raise GenericConfigUpdaterError("Failed to get cmd output '{}':rc {}".format(cmd, rc)) - try: - config_json = json.loads(stdout) - except json.JSONDecodeError as e: - raise GenericConfigUpdaterError("Failed to get config by '{}' due to {}".format(cmd, e)) - return config_json - - class GenericConfigUpdaterError(Exception): pass diff --git a/tests/generic_config_updater/multiasic_change_applier_test.py b/tests/generic_config_updater/multiasic_change_applier_test.py index cbeb0ef7b9..d7f734d2ec 100644 --- a/tests/generic_config_updater/multiasic_change_applier_test.py +++ b/tests/generic_config_updater/multiasic_change_applier_test.py @@ -38,7 +38,7 @@ def test_extract_scope(self): except Exception as e: assert(result == False) - @patch('generic_config_updater.change_applier._get_running_config', autospec=True) + @patch('generic_config_updater.change_applier.ChangeApplier._get_running_config', autospec=True) @patch('generic_config_updater.change_applier.ConfigDBConnector', autospec=True) def test_apply_change_default_scope(self, mock_ConfigDBConnector, mock_get_running_config): # Setup mock for ConfigDBConnector @@ -79,7 +79,7 @@ def test_apply_change_default_scope(self, mock_ConfigDBConnector, mock_get_runni # Assert ConfigDBConnector called with the correct namespace mock_ConfigDBConnector.assert_called_once_with(use_unix_socket_path=True, namespace="") - @patch('generic_config_updater.change_applier._get_running_config', autospec=True) + @patch('generic_config_updater.change_applier.ChangeApplier._get_running_config', autospec=True) @patch('generic_config_updater.change_applier.ConfigDBConnector', autospec=True) def test_apply_change_given_scope(self, mock_ConfigDBConnector, mock_get_running_config): # Setup mock for ConfigDBConnector @@ -120,7 +120,7 @@ def test_apply_change_given_scope(self, mock_ConfigDBConnector, mock_get_running # Assert ConfigDBConnector called with the correct scope mock_ConfigDBConnector.assert_called_once_with(use_unix_socket_path=True, namespace="asic0") - @patch('generic_config_updater.change_applier._get_running_config', autospec=True) + @patch('generic_config_updater.change_applier.ChangeApplier._get_running_config', autospec=True) @patch('generic_config_updater.change_applier.ConfigDBConnector', autospec=True) def test_apply_change_failure(self, mock_ConfigDBConnector, mock_get_running_config): # Setup mock for ConfigDBConnector @@ -142,7 +142,7 @@ def test_apply_change_failure(self, mock_ConfigDBConnector, mock_get_running_con self.assertTrue('Failed to get running config' in str(context.exception)) - @patch('generic_config_updater.change_applier._get_running_config', autospec=True) + @patch('generic_config_updater.change_applier.ChangeApplier._get_running_config', autospec=True) @patch('generic_config_updater.change_applier.ConfigDBConnector', autospec=True) def test_apply_patch_with_empty_tables_failure(self, mock_ConfigDBConnector, mock_get_running_config): # Setup mock for ConfigDBConnector From e469185e2a5f6763999be4356cc3f7db492a6e27 Mon Sep 17 00:00:00 2001 From: Xincun Li Date: Mon, 3 Jun 2024 14:48:37 -0700 Subject: [PATCH 37/44] fix scope --- generic_config_updater/gu_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generic_config_updater/gu_common.py b/generic_config_updater/gu_common.py index e68b4b5131..96072ab0fe 100644 --- a/generic_config_updater/gu_common.py +++ b/generic_config_updater/gu_common.py @@ -66,7 +66,7 @@ def get_config_db_as_json(self): return config_db_json def _get_config_db_as_text(self): - if self.namespace is not None and self.namespace != multi_asic.DEFAULT_NAMESPACE: + if self.scope is not None and self.scope != multi_asic.DEFAULT_NAMESPACE: cmd = ['sonic-cfggen', '-d', '--print-data', '-n', self.namespace] else: cmd = ['sonic-cfggen', '-d', '--print-data'] From 3e610daa555b1ad9a7df2649cfab4cd7f220fc29 Mon Sep 17 00:00:00 2001 From: Xincun Li Date: Mon, 3 Jun 2024 16:05:47 -0700 Subject: [PATCH 38/44] fix ut. --- generic_config_updater/generic_updater.py | 38 ++++++++++++++++++----- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/generic_config_updater/generic_updater.py b/generic_config_updater/generic_updater.py index 093f0ee655..cbcca9b338 100644 --- a/generic_config_updater/generic_updater.py +++ b/generic_config_updater/generic_updater.py @@ -254,10 +254,15 @@ def delete_checkpoint(self, checkpoint_name): class MultiASICConfigReplacer(ConfigReplacer): - def __init__(self): + def __init__(self, + patch_applier=None, + config_wrapper=None, + patch_wrapper=None, + scope=multi_asic.DEFAULT_NAMESPACE): self.logger = genericUpdaterLogging.get_logger(title="MultiASICConfigReplacer", print_all_to_console=True) self.scopelist = [HOST_NAMESPACE, *multi_asic.get_namespace_list()] + super().__init__(patch_applier, config_wrapper, patch_wrapper, scope) def replace(self, target_config): config_keys = set(target_config.keys()) @@ -273,12 +278,16 @@ def replace(self, target_config): class MultiASICConfigRollbacker(FileSystemConfigRollbacker): - def __init__(self, checkpoints_dir=CHECKPOINTS_DIR): + def __init__(self, + checkpoints_dir=CHECKPOINTS_DIR, + config_replacer=None, + config_wrapper=None): self.logger = genericUpdaterLogging.get_logger(title="MultiASICConfigRollbacker", print_all_to_console=True) self.scopelist = [HOST_NAMESPACE, *multi_asic.get_namespace_list()] self.checkpoints_dir = checkpoints_dir self.util = Util(checkpoints_dir=checkpoints_dir) + super().__init__(config_wrapper=config_wrapper, config_replacer=config_replacer) def rollback(self, checkpoint_name): self.logger.log_notice("Config rollbacking starting.") @@ -486,8 +495,13 @@ def create_config_replacer(self, config_format, verbose, dry_run, ignore_non_yan patch_wrapper=patch_wrapper, changeapplier=change_applier, scope=self.scope) - config_replacer = MultiASICConfigReplacer() if multi_asic.is_multi_asic() else ConfigReplacer( - patch_applier=patch_applier, config_wrapper=config_wrapper, scope=self.scope) + if multi_asic.is_multi_asic(): + config_replacer = MultiASICConfigReplacer(patch_applier=patch_applier, + config_wrapper=config_wrapper) + else: + config_replacer = ConfigReplacer(patch_applier=patch_applier, + config_wrapper=config_wrapper, + scope=self.scope) if config_format == ConfigFormat.CONFIGDB: pass @@ -515,10 +529,18 @@ def create_config_rollbacker(self, verbose, dry_run=False, ignore_non_yang_table patch_wrapper=patch_wrapper, changeapplier=change_applier, scope=self.scope) - config_replacer = MultiASICConfigReplacer() if multi_asic.is_multi_asic() else ConfigReplacer( - config_wrapper=config_wrapper, patch_applier=patch_applier, scope=self.scope) - config_rollbacker = MultiASICConfigRollbacker() if multi_asic.is_multi_asic() else FileSystemConfigRollbacker( - config_wrapper=config_wrapper, config_replacer=config_replacer, scope=self.scope) + if multi_asic.is_multi_asic(): + config_replacer = MultiASICConfigReplacer(config_wrapper=config_wrapper, + patch_applier=patch_applier) + config_rollbacker = MultiASICConfigRollbacker(config_wrapper=config_wrapper, + config_replacer=config_replacer) + else: + config_replacer = ConfigReplacer(config_wrapper=config_wrapper, + patch_applier=patch_applier, + scope=self.scope) + config_rollbacker = FileSystemConfigRollbacker(config_wrapper=config_wrapper, + config_replacer=config_replacer, + scope=self.scope) if not dry_run: config_rollbacker = ConfigLockDecorator(decorated_config_rollbacker=config_rollbacker, scope=self.scope) From acf663c9746ddef77f6d129db48c3136eb8c40c9 Mon Sep 17 00:00:00 2001 From: Xincun Li Date: Mon, 3 Jun 2024 17:30:10 -0700 Subject: [PATCH 39/44] improve coverage. --- tests/config_test.py | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/tests/config_test.py b/tests/config_test.py index b25807bf5a..4b351f50c1 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -3128,32 +3128,28 @@ def test_replace_multiasic_missing_scope(self): self.assertNotEqual(result.exit_code, 0, "Command should failed") self.assertIn("Failed to replace config", result.output) - @patch('generic_config_updater.gu_common.subprocess.Popen') + @patch('generic_config_updater.generic_updater.subprocess.Popen') @patch('generic_config_updater.generic_updater.Util.ensure_checkpoints_dir_exists', mock.Mock(return_value=True)) @patch('generic_config_updater.generic_updater.Util.save_json_file', MagicMock()) def test_checkpoint_multiasic(self, mock_subprocess_popen): allconfigs = copy.deepcopy(self.all_config) mock_instance = MagicMock() - mock_instance.communicate.side_effect = (allconfigs, 0) + mock_instance.communicate.return_value = (json.dumps(allconfigs), 0) mock_instance.returncode = 0 mock_subprocess_popen.return_value = mock_instance checkpointname = "checkpointname" - # Mock GenericUpdater to avoid actual patch application - with patch('config.main.GenericUpdater') as mock_generic_updater: - mock_generic_updater.return_value.checkpoint = MagicMock() - - print("Multi ASIC: {}".format(multi_asic.is_multi_asic())) - # Invocation of the command with the CliRunner - result = self.runner.invoke(config.config.commands["checkpoint"], - [checkpointname], - catch_exceptions=True) + print("Multi ASIC: {}".format(multi_asic.is_multi_asic())) + # Invocation of the command with the CliRunner + result = self.runner.invoke(config.config.commands["checkpoint"], + [checkpointname], + catch_exceptions=True) - print("Exit Code: {}, output: {}".format(result.exit_code, result.output)) - # Assertions and verifications - self.assertEqual(result.exit_code, 0, "Command should succeed") - self.assertIn("Checkpoint created successfully.", result.output) + print("Exit Code: {}, output: {}".format(result.exit_code, result.output)) + # Assertions and verifications + self.assertEqual(result.exit_code, 0, "Command should succeed") + self.assertIn("Checkpoint created successfully.", result.output) @patch('generic_config_updater.generic_updater.Util.check_checkpoint_exists', mock.Mock(return_value=True)) @patch('generic_config_updater.generic_updater.ConfigReplacer.replace', MagicMock()) From ff036e7f35bacb80b16b0fa44863b294b17d1671 Mon Sep 17 00:00:00 2001 From: Xincun Li Date: Wed, 5 Jun 2024 11:46:54 -0700 Subject: [PATCH 40/44] fix replacer --- generic_config_updater/generic_updater.py | 2 +- generic_config_updater/gu_common.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/generic_config_updater/generic_updater.py b/generic_config_updater/generic_updater.py index cbcca9b338..452c34db66 100644 --- a/generic_config_updater/generic_updater.py +++ b/generic_config_updater/generic_updater.py @@ -274,7 +274,7 @@ def replace(self, target_config): scope_config = target_config.pop(scope) if scope.lower() == HOST_NAMESPACE: scope = multi_asic.DEFAULT_NAMESPACE - super(scope=scope).replace(scope_config) + ConfigReplacer(scope=scope).replace(scope_config) class MultiASICConfigRollbacker(FileSystemConfigRollbacker): diff --git a/generic_config_updater/gu_common.py b/generic_config_updater/gu_common.py index 96072ab0fe..2870542bd6 100644 --- a/generic_config_updater/gu_common.py +++ b/generic_config_updater/gu_common.py @@ -67,7 +67,7 @@ def get_config_db_as_json(self): def _get_config_db_as_text(self): if self.scope is not None and self.scope != multi_asic.DEFAULT_NAMESPACE: - cmd = ['sonic-cfggen', '-d', '--print-data', '-n', self.namespace] + cmd = ['sonic-cfggen', '-d', '--print-data', '-n', self.scope] else: cmd = ['sonic-cfggen', '-d', '--print-data'] @@ -75,7 +75,7 @@ def _get_config_db_as_text(self): text, err = result.communicate() return_code = result.returncode if return_code: # non-zero means failure - raise GenericConfigUpdaterError(f"Failed to get running config for namespace: {self.namespace}, Return code: {return_code}, Error: {err}") + raise GenericConfigUpdaterError(f"Failed to get running config for namespace: {self.scope}, Return code: {return_code}, Error: {err}") return text def get_sonic_yang_as_json(self): From 35690374c89881bfdbf8ace81e8b957dce44550e Mon Sep 17 00:00:00 2001 From: Xincun Li Date: Wed, 5 Jun 2024 11:49:10 -0700 Subject: [PATCH 41/44] fix format. --- generic_config_updater/gu_common.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/generic_config_updater/gu_common.py b/generic_config_updater/gu_common.py index 2870542bd6..938aa1d034 100644 --- a/generic_config_updater/gu_common.py +++ b/generic_config_updater/gu_common.py @@ -75,7 +75,8 @@ def _get_config_db_as_text(self): text, err = result.communicate() return_code = result.returncode if return_code: # non-zero means failure - raise GenericConfigUpdaterError(f"Failed to get running config for namespace: {self.scope}, Return code: {return_code}, Error: {err}") + raise GenericConfigUpdaterError(f"Failed to get running config for namespace: {self.scope}," + f" Return code: {return_code}, Error: {err}") return text def get_sonic_yang_as_json(self): From fbbc34ecccd0a8c5464406854bdf15f380032324 Mon Sep 17 00:00:00 2001 From: Xincun Li Date: Wed, 5 Jun 2024 15:08:40 -0700 Subject: [PATCH 42/44] Remove unused parts --- config/main.py | 1 - generic_config_updater/change_applier.py | 7 ++----- generic_config_updater/generic_updater.py | 2 +- tests/config_test.py | 4 ---- 4 files changed, 3 insertions(+), 11 deletions(-) diff --git a/config/main.py b/config/main.py index c9ae919a72..8bd6742051 100644 --- a/config/main.py +++ b/config/main.py @@ -1522,7 +1522,6 @@ def replace(ctx, target_file_path, format, dry_run, ignore_non_yang_tables, igno click.secho("Failed to replace config", fg="red", underline=True, err=True) ctx.fail(ex) - @config.command() @click.argument('checkpoint-name', type=str, required=True) @click.option('-d', '--dry-run', is_flag=True, default=False, help='test out the command without affecting config state') diff --git a/generic_config_updater/change_applier.py b/generic_config_updater/change_applier.py index f41835945b..8d8d23f87a 100644 --- a/generic_config_updater/change_applier.py +++ b/generic_config_updater/change_applier.py @@ -16,6 +16,7 @@ print_to_console = False + def set_verbose(verbose=False): global print_to_console, logger @@ -39,6 +40,7 @@ def get_config_db(scope=multi_asic.DEFAULT_NAMESPACE): config_db.connect() return config_db + def set_config(config_db, tbl, key, data): config_db.set_entry(tbl, key, data) @@ -61,11 +63,9 @@ class DryRunChangeApplier: def __init__(self, config_wrapper): self.config_wrapper = config_wrapper - def apply(self, change): self.config_wrapper.apply_change_to_config_db(change) - def remove_backend_tables_from_config(self, data): return data @@ -86,7 +86,6 @@ def __init__(self, scope=multi_asic.DEFAULT_NAMESPACE): with open(UPDATER_CONF_FILE, "r") as s: ChangeApplier.updater_conf = json.load(s) - def _invoke_cmd(self, cmd, old_cfg, upd_cfg, keys): # cmd is in the format as . # @@ -98,7 +97,6 @@ def _invoke_cmd(self, cmd, old_cfg, upd_cfg, keys): return method_to_call(old_cfg, upd_cfg, keys) - def _services_validate(self, old_cfg, upd_cfg, keys): lst_svcs = set() lst_cmds = set() @@ -124,7 +122,6 @@ def _services_validate(self, old_cfg, upd_cfg, keys): log_debug("service invoked: {}".format(cmd)) return 0 - def _upd_data(self, tbl, run_tbl, upd_tbl, upd_keys): for key in set(run_tbl.keys()).union(set(upd_tbl.keys())): run_data = run_tbl.get(key, None) diff --git a/generic_config_updater/generic_updater.py b/generic_config_updater/generic_updater.py index 452c34db66..d83f0f3057 100644 --- a/generic_config_updater/generic_updater.py +++ b/generic_config_updater/generic_updater.py @@ -1,7 +1,7 @@ import json -import subprocess import jsonpointer import os +import subprocess from enum import Enum from .gu_common import HOST_NAMESPACE, GenericConfigUpdaterError, EmptyTableError, ConfigWrapper, \ diff --git a/tests/config_test.py b/tests/config_test.py index 4b351f50c1..a505bc2209 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -1421,7 +1421,6 @@ def test_replace__help__gets_help_msg(self): self.assertEqual(expected_exit_code, result.exit_code) self.assertTrue(expected_output in result.output) - @patch('config.main.SonicYangCfgDbGenerator.validate_config_db_json', mock.Mock(return_value=True)) def test_replace__only_required_params__default_values_used_for_optional_params(self): # Arrange expected_exit_code = 0 @@ -1440,7 +1439,6 @@ def test_replace__only_required_params__default_values_used_for_optional_params( mock_generic_updater.replace.assert_called_once() mock_generic_updater.replace.assert_has_calls([expected_call_with_default_values]) - @patch('config.main.SonicYangCfgDbGenerator.validate_config_db_json', mock.Mock(return_value=True)) def test_replace__all_optional_params_non_default__non_default_values_used(self): # Arrange expected_exit_code = 0 @@ -1470,7 +1468,6 @@ def test_replace__all_optional_params_non_default__non_default_values_used(self) mock_generic_updater.replace.assert_called_once() mock_generic_updater.replace.assert_has_calls([expected_call_with_non_default_values]) - @patch('config.main.SonicYangCfgDbGenerator.validate_config_db_json', mock.Mock(return_value=True)) def test_replace__exception_thrown__error_displayed_error_code_returned(self): # Arrange unexpected_exit_code = 0 @@ -1489,7 +1486,6 @@ def test_replace__exception_thrown__error_displayed_error_code_returned(self): self.assertNotEqual(unexpected_exit_code, result.exit_code) self.assertTrue(any_error_message in result.output) - @patch('config.main.SonicYangCfgDbGenerator.validate_config_db_json', mock.Mock(return_value=True)) def test_replace__optional_parameters_passed_correctly(self): self.validate_replace_optional_parameter( ["--format", ConfigFormat.SONICYANG.name], From c4d20c60d94ca56ed55d0b1a4bd73ee3115e1fe7 Mon Sep 17 00:00:00 2001 From: Xincun Li Date: Thu, 6 Jun 2024 10:20:04 -0700 Subject: [PATCH 43/44] add get_config_db_as_text ut --- .../generic_config_updater/gu_common_test.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/generic_config_updater/gu_common_test.py b/tests/generic_config_updater/gu_common_test.py index a2a776c0bb..4a16a5ca4f 100644 --- a/tests/generic_config_updater/gu_common_test.py +++ b/tests/generic_config_updater/gu_common_test.py @@ -76,6 +76,28 @@ def test_ctor__default_values_set(self): self.assertEqual("/usr/local/yang-models", gu_common.YANG_DIR) + @patch('generic_config_updater.gu_common.subprocess.Popen') + def test_get_config_db_as_text(self, mock_popen): + config_wrapper = gu_common.ConfigWrapper() + mock_proc = MagicMock() + mock_proc.communicate = MagicMock( + return_value=("[]", None)) + mock_proc.returncode = 0 + mock_popen.return_value = mock_proc + actual = config_wrapper._get_config_db_as_text() + expected = "[]" + self.assertEqual(actual, expected) + + config_wrapper = gu_common.ConfigWrapper(scope="asic0") + mock_proc = MagicMock() + mock_proc.communicate = MagicMock( + return_value=("[]", None)) + mock_proc.returncode = 0 + mock_popen.return_value = mock_proc + actual = config_wrapper._get_config_db_as_text() + expected = "[]" + self.assertEqual(actual, expected) + def test_get_sonic_yang_as_json__returns_sonic_yang_as_json(self): # Arrange config_wrapper = self.config_wrapper_mock From 05cb10f8869e351167c746554c1c2f4560a4fe8a Mon Sep 17 00:00:00 2001 From: Xincun Li Date: Wed, 12 Jun 2024 11:36:47 -0700 Subject: [PATCH 44/44] Change get_config_json implementation --- generic_config_updater/generic_updater.py | 31 +++++++++++++++-------- tests/config_test.py | 29 ++++++++++++++++++--- 2 files changed, 45 insertions(+), 15 deletions(-) diff --git a/generic_config_updater/generic_updater.py b/generic_config_updater/generic_updater.py index d83f0f3057..b6d65e2ce6 100644 --- a/generic_config_updater/generic_updater.py +++ b/generic_config_updater/generic_updater.py @@ -42,18 +42,27 @@ def get_cmd_output(cmd): def get_config_json(): - command = ["show", "runningconfiguration", "all"] - all_running_config_text, returncode = get_cmd_output(command) - if returncode: - raise GenericConfigUpdaterError( - f"Fetch all runningconfiguration failed as output:{all_running_config_text}") - all_running_config = json.loads(all_running_config_text) - + scope_list = [multi_asic.DEFAULT_NAMESPACE] + all_running_config = {} if multi_asic.is_multi_asic(): - for asic in [HOST_NAMESPACE, *multi_asic.get_namespace_list()]: - all_running_config[asic].pop("bgpraw", None) - else: - all_running_config.pop("bgpraw", None) + scope_list.extend(multi_asic.get_namespace_list()) + for scope in scope_list: + command = ["sonic-cfggen", "-d", "--print-data"] + if scope != multi_asic.DEFAULT_NAMESPACE: + command += ["-n", scope] + + running_config_text, returncode = get_cmd_output(command) + if returncode: + raise GenericConfigUpdaterError( + f"Fetch all runningconfiguration failed as output:{running_config_text}") + running_config = json.loads(running_config_text) + + if multi_asic.is_multi_asic(): + if scope == multi_asic.DEFAULT_NAMESPACE: + scope = HOST_NAMESPACE + all_running_config[scope] = running_config + else: + all_running_config = running_config return all_running_config diff --git a/tests/config_test.py b/tests/config_test.py index a505bc2209..670932a436 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -3130,10 +3130,31 @@ def test_replace_multiasic_missing_scope(self): def test_checkpoint_multiasic(self, mock_subprocess_popen): allconfigs = copy.deepcopy(self.all_config) - mock_instance = MagicMock() - mock_instance.communicate.return_value = (json.dumps(allconfigs), 0) - mock_instance.returncode = 0 - mock_subprocess_popen.return_value = mock_instance + # Create mock instances for each subprocess call + mock_instance_localhost = MagicMock() + mock_instance_localhost.communicate.return_value = (json.dumps(allconfigs["localhost"]), 0) + mock_instance_localhost.returncode = 0 + + mock_instance_asic0 = MagicMock() + mock_instance_asic0.communicate.return_value = (json.dumps(allconfigs["asic0"]), 0) + mock_instance_asic0.returncode = 0 + + mock_instance_asic1 = MagicMock() + mock_instance_asic1.communicate.return_value = (json.dumps(allconfigs["asic1"]), 0) + mock_instance_asic1.returncode = 0 + + # Setup side effect to return different mock instances based on input arguments + def side_effect(*args, **kwargs): + if "asic" not in args[0]: + return mock_instance_localhost + elif "asic0" in args[0]: + return mock_instance_asic0 + elif "asic1" in args[0]: + return mock_instance_asic1 + else: + return MagicMock() # Default case + + mock_subprocess_popen.side_effect = side_effect checkpointname = "checkpointname" print("Multi ASIC: {}".format(multi_asic.is_multi_asic()))