From 970fc749ef995d919307586b44ad222245d277e0 Mon Sep 17 00:00:00 2001 From: David Emerson <43939939+emersodb@users.noreply.github.com> Date: Thu, 17 Oct 2024 17:45:03 -0400 Subject: [PATCH] re-enabling the remainder of the smoke tests --- tests/smoke_tests/run_smoke_test.py | 422 ++++++++++++++-------------- 1 file changed, 211 insertions(+), 211 deletions(-) diff --git a/tests/smoke_tests/run_smoke_test.py b/tests/smoke_tests/run_smoke_test.py index 9db0d714..14201152 100644 --- a/tests/smoke_tests/run_smoke_test.py +++ b/tests/smoke_tests/run_smoke_test.py @@ -627,67 +627,67 @@ def load_metrics_from_file(file_path: str) -> Dict[str, Any]: if __name__ == "__main__": loop = asyncio.get_event_loop() - # loop.run_until_complete( - # run_fault_tolerance_smoke_test( - # server_python_path="tests.smoke_tests.load_from_checkpoint_example.server", - # client_python_path="tests.smoke_tests.load_from_checkpoint_example.client", - # config_path="tests/smoke_tests/load_from_checkpoint_example/config.yaml", - # partial_config_path="tests/smoke_tests/load_from_checkpoint_example/partial_config.yaml", - # dataset_path="examples/datasets/cifar_data/", - # seed=42, - # server_metrics=load_metrics_from_file("tests/smoke_tests/basic_server_metrics.json"), - # client_metrics=load_metrics_from_file("tests/smoke_tests/basic_client_metrics.json"), - # ) - # ) - # loop.run_until_complete( - # run_smoke_test( # By default will use Task04_Hippocampus Dataset - # server_python_path="examples.nnunet_example.server", - # client_python_path="examples.nnunet_example.client", - # config_path="tests/smoke_tests/nnunet_config_2d.yaml", - # dataset_path="examples/datasets/nnunet", - # ) - # ) - # loop.run_until_complete( - # run_smoke_test( # By default will use Task04_Hippocampus Dataset - # server_python_path="examples.nnunet_example.server", - # client_python_path="examples.nnunet_example.client", - # config_path="tests/smoke_tests/nnunet_config_3d.yaml", - # dataset_path="examples/datasets/nnunet", - # ) - # ) - # loop.run_until_complete( - # run_smoke_test( - # server_python_path="examples.fedprox_example.server", - # client_python_path="examples.fedprox_example.client", - # config_path="tests/smoke_tests/fedprox_config.yaml", - # dataset_path="examples/datasets/mnist_data/", - # seed=42, - # server_metrics=load_metrics_from_file("tests/smoke_tests/fedprox_server_metrics.json"), - # client_metrics=load_metrics_from_file("tests/smoke_tests/fedprox_client_metrics.json"), - # ) - # ) - # loop.run_until_complete( - # run_smoke_test( - # server_python_path="examples.scaffold_example.server", - # client_python_path="examples.scaffold_example.client", - # config_path="tests/smoke_tests/scaffold_config.yaml", - # dataset_path="examples/datasets/mnist_data/", - # seed=42, - # server_metrics=load_metrics_from_file("tests/smoke_tests/scaffold_server_metrics.json"), - # client_metrics=load_metrics_from_file("tests/smoke_tests/scaffold_client_metrics.json"), - # ) - # ) - # loop.run_until_complete( - # run_smoke_test( - # server_python_path="examples.apfl_example.server", - # client_python_path="examples.apfl_example.client", - # config_path="tests/smoke_tests/apfl_config.yaml", - # dataset_path="examples/datasets/mnist_data/", - # seed=42, - # server_metrics=load_metrics_from_file("tests/smoke_tests/apfl_server_metrics.json"), - # client_metrics=load_metrics_from_file("tests/smoke_tests/apfl_client_metrics.json"), - # ) - # ) + loop.run_until_complete( + run_fault_tolerance_smoke_test( + server_python_path="tests.smoke_tests.load_from_checkpoint_example.server", + client_python_path="tests.smoke_tests.load_from_checkpoint_example.client", + config_path="tests/smoke_tests/load_from_checkpoint_example/config.yaml", + partial_config_path="tests/smoke_tests/load_from_checkpoint_example/partial_config.yaml", + dataset_path="examples/datasets/cifar_data/", + seed=42, + server_metrics=load_metrics_from_file("tests/smoke_tests/basic_server_metrics.json"), + client_metrics=load_metrics_from_file("tests/smoke_tests/basic_client_metrics.json"), + ) + ) + loop.run_until_complete( + run_smoke_test( # By default will use Task04_Hippocampus Dataset + server_python_path="examples.nnunet_example.server", + client_python_path="examples.nnunet_example.client", + config_path="tests/smoke_tests/nnunet_config_2d.yaml", + dataset_path="examples/datasets/nnunet", + ) + ) + loop.run_until_complete( + run_smoke_test( # By default will use Task04_Hippocampus Dataset + server_python_path="examples.nnunet_example.server", + client_python_path="examples.nnunet_example.client", + config_path="tests/smoke_tests/nnunet_config_3d.yaml", + dataset_path="examples/datasets/nnunet", + ) + ) + loop.run_until_complete( + run_smoke_test( + server_python_path="examples.fedprox_example.server", + client_python_path="examples.fedprox_example.client", + config_path="tests/smoke_tests/fedprox_config.yaml", + dataset_path="examples/datasets/mnist_data/", + seed=42, + server_metrics=load_metrics_from_file("tests/smoke_tests/fedprox_server_metrics.json"), + client_metrics=load_metrics_from_file("tests/smoke_tests/fedprox_client_metrics.json"), + ) + ) + loop.run_until_complete( + run_smoke_test( + server_python_path="examples.scaffold_example.server", + client_python_path="examples.scaffold_example.client", + config_path="tests/smoke_tests/scaffold_config.yaml", + dataset_path="examples/datasets/mnist_data/", + seed=42, + server_metrics=load_metrics_from_file("tests/smoke_tests/scaffold_server_metrics.json"), + client_metrics=load_metrics_from_file("tests/smoke_tests/scaffold_client_metrics.json"), + ) + ) + loop.run_until_complete( + run_smoke_test( + server_python_path="examples.apfl_example.server", + client_python_path="examples.apfl_example.client", + config_path="tests/smoke_tests/apfl_config.yaml", + dataset_path="examples/datasets/mnist_data/", + seed=42, + server_metrics=load_metrics_from_file("tests/smoke_tests/apfl_server_metrics.json"), + client_metrics=load_metrics_from_file("tests/smoke_tests/apfl_client_metrics.json"), + ) + ) loop.run_until_complete( run_smoke_test( server_python_path="examples.feddg_ga_example.server", @@ -699,154 +699,154 @@ def load_metrics_from_file(file_path: str) -> Dict[str, Any]: client_metrics=load_metrics_from_file("tests/smoke_tests/feddg_ga_client_metrics.json"), ) ) - # loop.run_until_complete( - # run_smoke_test( - # server_python_path="examples.basic_example.server", - # client_python_path="examples.basic_example.client", - # config_path="tests/smoke_tests/basic_config.yaml", - # dataset_path="examples/datasets/cifar_data/", - # ) - # ) - # loop.run_until_complete( - # run_smoke_test( - # server_python_path="examples.dp_fed_examples.client_level_dp.server", - # client_python_path="examples.dp_fed_examples.client_level_dp.client", - # config_path="tests/smoke_tests/client_level_dp_config.yaml", - # dataset_path="examples/datasets/cifar_data/", - # skip_assert_client_fl_rounds=True, - # ) - # ) - # loop.run_until_complete( - # run_smoke_test( - # server_python_path="examples.dp_fed_examples.client_level_dp_weighted.server", - # client_python_path="examples.dp_fed_examples.client_level_dp_weighted.client", - # config_path="tests/smoke_tests/client_level_dp_weighted_config.yaml", - # dataset_path="examples/datasets/breast_cancer_data/hospital_0.csv", - # skip_assert_client_fl_rounds=True, - # ) - # ) - # loop.run_until_complete( - # run_smoke_test( - # server_python_path="examples.dp_fed_examples.instance_level_dp.server", - # client_python_path="examples.dp_fed_examples.instance_level_dp.client", - # config_path="tests/smoke_tests/instance_level_dp_config.yaml", - # dataset_path="examples/datasets/cifar_data/", - # skip_assert_client_fl_rounds=True, - # ) - # ) - # loop.run_until_complete( - # run_smoke_test( - # server_python_path="examples.dp_scaffold_example.server", - # client_python_path="examples.dp_scaffold_example.client", - # config_path="tests/smoke_tests/dp_scaffold_config.yaml", - # dataset_path="examples/datasets/mnist_data/", - # ) - # ) - # loop.run_until_complete( - # run_smoke_test( - # server_python_path="examples.fedbn_example.server", - # client_python_path="examples.fedbn_example.client", - # config_path="tests/smoke_tests/fedbn_config.yaml", - # dataset_path="examples/datasets/mnist_data/", - # ) - # ) - # loop.run_until_complete( - # run_smoke_test( - # server_python_path="examples.federated_eval_example.server", - # client_python_path="examples.federated_eval_example.client", - # config_path="tests/smoke_tests/federated_eval_config.yaml", - # dataset_path="examples/datasets/cifar_data/", - # checkpoint_path="examples/assets/best_checkpoint_fczjmljm.pkl", - # assert_evaluation_logs=True, - # ) - # ) - # loop.run_until_complete( - # run_smoke_test( - # server_python_path="examples.fedper_example.server", - # client_python_path="examples.fedper_example.client", - # config_path="tests/smoke_tests/fedper_config.yaml", - # dataset_path="examples/datasets/mnist_data/", - # ) - # ) - # loop.run_until_complete( - # run_smoke_test( - # server_python_path="examples.fedrep_example.server", - # client_python_path="examples.fedrep_example.client", - # config_path="tests/smoke_tests/fedrep_config.yaml", - # dataset_path="examples/datasets/cifar_data/", - # ) - # ) - # loop.run_until_complete( - # run_smoke_test( - # server_python_path="examples.ditto_example.server", - # client_python_path="examples.ditto_example.client", - # config_path="tests/smoke_tests/ditto_config.yaml", - # dataset_path="examples/datasets/mnist_data/", - # ) - # ) - # loop.run_until_complete( - # run_smoke_test( - # server_python_path="examples.mr_mtl_example.server", - # client_python_path="examples.mr_mtl_example.client", - # config_path="tests/smoke_tests/mr_mtl_config.yaml", - # dataset_path="examples/datasets/mnist_data/", - # ) - # ) - # loop.run_until_complete( - # run_smoke_test( - # server_python_path="examples.fenda_example.server", - # client_python_path="examples.fenda_example.client", - # config_path="tests/smoke_tests/fenda_config.yaml", - # dataset_path="examples/datasets/mnist_data/", - # ) - # ) - # loop.run_until_complete( - # run_smoke_test( - # server_python_path="examples.fenda_ditto_example.server", - # client_python_path="examples.fenda_ditto_example.client", - # config_path="tests/smoke_tests/fenda_ditto_config.yaml", - # dataset_path="examples/datasets/mnist_data/", - # checkpoint_path="examples/assets/", - # ) - # ) - # loop.run_until_complete( - # run_smoke_test( - # server_python_path="examples.perfcl_example.server", - # client_python_path="examples.perfcl_example.client", - # config_path="tests/smoke_tests/perfcl_config.yaml", - # dataset_path="examples/datasets/mnist_data/", - # ) - # ) - # loop.run_until_complete( - # run_smoke_test( - # server_python_path="examples.fl_plus_local_ft_example.server", - # client_python_path="examples.fl_plus_local_ft_example.client", - # config_path="tests/smoke_tests/fl_plus_local_ft_config.yaml", - # dataset_path="examples/datasets/cifar_data/", - # ) - # ) - # loop.run_until_complete( - # run_smoke_test( - # server_python_path="examples.moon_example.server", - # client_python_path="examples.moon_example.client", - # config_path="tests/smoke_tests/moon_config.yaml", - # dataset_path="examples/datasets/mnist_data/", - # ) - # ) - # loop.run_until_complete( - # run_smoke_test( - # server_python_path="examples.ensemble_example.server", - # client_python_path="examples.ensemble_example.client", - # config_path="tests/smoke_tests/ensemble_config.yaml", - # dataset_path="examples/datasets/mnist_data/", - # ) - # ) - # loop.run_until_complete( - # run_smoke_test( - # server_python_path="examples.flash_example.server", - # client_python_path="examples.flash_example.client", - # config_path="tests/smoke_tests/flash_config.yaml", - # dataset_path="examples/datasets/cifar_data/", - # ) - # ) + loop.run_until_complete( + run_smoke_test( + server_python_path="examples.basic_example.server", + client_python_path="examples.basic_example.client", + config_path="tests/smoke_tests/basic_config.yaml", + dataset_path="examples/datasets/cifar_data/", + ) + ) + loop.run_until_complete( + run_smoke_test( + server_python_path="examples.dp_fed_examples.client_level_dp.server", + client_python_path="examples.dp_fed_examples.client_level_dp.client", + config_path="tests/smoke_tests/client_level_dp_config.yaml", + dataset_path="examples/datasets/cifar_data/", + skip_assert_client_fl_rounds=True, + ) + ) + loop.run_until_complete( + run_smoke_test( + server_python_path="examples.dp_fed_examples.client_level_dp_weighted.server", + client_python_path="examples.dp_fed_examples.client_level_dp_weighted.client", + config_path="tests/smoke_tests/client_level_dp_weighted_config.yaml", + dataset_path="examples/datasets/breast_cancer_data/hospital_0.csv", + skip_assert_client_fl_rounds=True, + ) + ) + loop.run_until_complete( + run_smoke_test( + server_python_path="examples.dp_fed_examples.instance_level_dp.server", + client_python_path="examples.dp_fed_examples.instance_level_dp.client", + config_path="tests/smoke_tests/instance_level_dp_config.yaml", + dataset_path="examples/datasets/cifar_data/", + skip_assert_client_fl_rounds=True, + ) + ) + loop.run_until_complete( + run_smoke_test( + server_python_path="examples.dp_scaffold_example.server", + client_python_path="examples.dp_scaffold_example.client", + config_path="tests/smoke_tests/dp_scaffold_config.yaml", + dataset_path="examples/datasets/mnist_data/", + ) + ) + loop.run_until_complete( + run_smoke_test( + server_python_path="examples.fedbn_example.server", + client_python_path="examples.fedbn_example.client", + config_path="tests/smoke_tests/fedbn_config.yaml", + dataset_path="examples/datasets/mnist_data/", + ) + ) + loop.run_until_complete( + run_smoke_test( + server_python_path="examples.federated_eval_example.server", + client_python_path="examples.federated_eval_example.client", + config_path="tests/smoke_tests/federated_eval_config.yaml", + dataset_path="examples/datasets/cifar_data/", + checkpoint_path="examples/assets/best_checkpoint_fczjmljm.pkl", + assert_evaluation_logs=True, + ) + ) + loop.run_until_complete( + run_smoke_test( + server_python_path="examples.fedper_example.server", + client_python_path="examples.fedper_example.client", + config_path="tests/smoke_tests/fedper_config.yaml", + dataset_path="examples/datasets/mnist_data/", + ) + ) + loop.run_until_complete( + run_smoke_test( + server_python_path="examples.fedrep_example.server", + client_python_path="examples.fedrep_example.client", + config_path="tests/smoke_tests/fedrep_config.yaml", + dataset_path="examples/datasets/cifar_data/", + ) + ) + loop.run_until_complete( + run_smoke_test( + server_python_path="examples.ditto_example.server", + client_python_path="examples.ditto_example.client", + config_path="tests/smoke_tests/ditto_config.yaml", + dataset_path="examples/datasets/mnist_data/", + ) + ) + loop.run_until_complete( + run_smoke_test( + server_python_path="examples.mr_mtl_example.server", + client_python_path="examples.mr_mtl_example.client", + config_path="tests/smoke_tests/mr_mtl_config.yaml", + dataset_path="examples/datasets/mnist_data/", + ) + ) + loop.run_until_complete( + run_smoke_test( + server_python_path="examples.fenda_example.server", + client_python_path="examples.fenda_example.client", + config_path="tests/smoke_tests/fenda_config.yaml", + dataset_path="examples/datasets/mnist_data/", + ) + ) + loop.run_until_complete( + run_smoke_test( + server_python_path="examples.fenda_ditto_example.server", + client_python_path="examples.fenda_ditto_example.client", + config_path="tests/smoke_tests/fenda_ditto_config.yaml", + dataset_path="examples/datasets/mnist_data/", + checkpoint_path="examples/assets/", + ) + ) + loop.run_until_complete( + run_smoke_test( + server_python_path="examples.perfcl_example.server", + client_python_path="examples.perfcl_example.client", + config_path="tests/smoke_tests/perfcl_config.yaml", + dataset_path="examples/datasets/mnist_data/", + ) + ) + loop.run_until_complete( + run_smoke_test( + server_python_path="examples.fl_plus_local_ft_example.server", + client_python_path="examples.fl_plus_local_ft_example.client", + config_path="tests/smoke_tests/fl_plus_local_ft_config.yaml", + dataset_path="examples/datasets/cifar_data/", + ) + ) + loop.run_until_complete( + run_smoke_test( + server_python_path="examples.moon_example.server", + client_python_path="examples.moon_example.client", + config_path="tests/smoke_tests/moon_config.yaml", + dataset_path="examples/datasets/mnist_data/", + ) + ) + loop.run_until_complete( + run_smoke_test( + server_python_path="examples.ensemble_example.server", + client_python_path="examples.ensemble_example.client", + config_path="tests/smoke_tests/ensemble_config.yaml", + dataset_path="examples/datasets/mnist_data/", + ) + ) + loop.run_until_complete( + run_smoke_test( + server_python_path="examples.flash_example.server", + client_python_path="examples.flash_example.client", + config_path="tests/smoke_tests/flash_config.yaml", + dataset_path="examples/datasets/cifar_data/", + ) + ) loop.close()