From 6ca9bad15bb06aea1477cf3c51dd376dd3bf95c6 Mon Sep 17 00:00:00 2001 From: liupengcheng Date: Sat, 8 Apr 2023 17:36:50 +0800 Subject: [PATCH] =?UTF-8?q?=E4=B8=8A=E4=BC=A0kineto=20=E7=89=88=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../.github/workflows/libkineto_ci.yml | 56 + .../workflows/tb_plugin_build_pip_package.yml | 19 + .../.github/workflows/tb_plugin_ci.yml | 57 + tb_plugins/profiling/.gitignore | 3 + tb_plugins/profiling/.gitmodules | 6 + tb_plugins/profiling/CODE_OF_CONDUCT.md | 77 + tb_plugins/profiling/CONTRIBUTING.md | 34 + tb_plugins/profiling/LICENSE | 33 + tb_plugins/profiling/README.md | 38 + tb_plugins/profiling/libkineto/CMakeLists.txt | 198 + tb_plugins/profiling/libkineto/README.md | 65 + .../libkineto/include/AbstractConfig.h | 113 + .../include/ActivityProfilerInterface.h | 91 + .../include/ActivityTraceInterface.h | 21 + .../libkineto/include/ActivityType.h | 34 + .../libkineto/include/ClientInterface.h | 16 + .../profiling/libkineto/include/Config.h | 433 + .../libkineto/include/GenericTraceActivity.h | 125 + .../libkineto/include/IActivityProfiler.h | 104 + .../libkineto/include/ILoggerObserver.h | 50 + .../libkineto/include/ITraceActivity.h | 53 + .../profiling/libkineto/include/ThreadUtil.h | 22 + .../profiling/libkineto/include/TraceSpan.h | 36 + .../profiling/libkineto/include/libkineto.h | 138 + .../libkineto/include/time_since_epoch.h | 16 + .../profiling/libkineto/libkineto_defs.bzl | 77 + .../sample_programs/kineto_playground.cpp | 38 + .../sample_programs/kineto_playground.cu | 60 + .../sample_programs/kineto_playground.cuh | 18 + .../libkineto/src/AbstractConfig.cpp | 188 + .../profiling/libkineto/src/ActivityBuffers.h | 29 + .../libkineto/src/ActivityLoggerFactory.h | 60 + .../src/ActivityProfilerController.cpp | 246 + .../src/ActivityProfilerController.h | 84 + .../libkineto/src/ActivityProfilerProxy.cpp | 119 + .../libkineto/src/ActivityProfilerProxy.h | 73 + .../profiling/libkineto/src/ActivityTrace.h | 45 + .../profiling/libkineto/src/ActivityType.cpp | 58 + tb_plugins/profiling/libkineto/src/Config.cpp | 473 + .../profiling/libkineto/src/ConfigLoader.cpp | 300 + .../profiling/libkineto/src/ConfigLoader.h | 147 + .../libkineto/src/CudaDeviceProperties.cpp | 130 + .../libkineto/src/CudaDeviceProperties.h | 31 + .../profiling/libkineto/src/CuptiActivity.h | 114 + .../profiling/libkineto/src/CuptiActivity.tpp | 111 + .../libkineto/src/CuptiActivityApi.cpp | 343 + .../libkineto/src/CuptiActivityApi.h | 100 + .../libkineto/src/CuptiActivityBuffer.h | 51 + .../libkineto/src/CuptiActivityPlatform.cpp | 31 + .../libkineto/src/CuptiActivityPlatform.h | 12 + .../libkineto/src/CuptiActivityProfiler.cpp | 841 ++ .../libkineto/src/CuptiActivityProfiler.h | 364 + .../libkineto/src/CuptiCallbackApi.cpp | 260 + .../libkineto/src/CuptiCallbackApi.h | 130 + .../libkineto/src/CuptiCallbackApiMock.h | 32 + .../profiling/libkineto/src/CuptiEventApi.cpp | 112 + .../profiling/libkineto/src/CuptiEventApi.h | 49 + .../libkineto/src/CuptiMetricApi.cpp | 107 + .../profiling/libkineto/src/CuptiMetricApi.h | 38 + .../libkineto/src/CuptiNvPerfMetric.cpp | 504 + .../libkineto/src/CuptiNvPerfMetric.h | 71 + .../libkineto/src/CuptiRangeProfilerApi.cpp | 751 ++ .../libkineto/src/CuptiRangeProfilerApi.h | 220 + .../src/CuptiRangeProfilerConfig.cpp | 68 + .../libkineto/src/CuptiRangeProfilerConfig.h | 86 + .../libkineto/src/DaemonConfigLoader.h | 27 + .../profiling/libkineto/src/Demangle.cpp | 49 + tb_plugins/profiling/libkineto/src/Demangle.h | 12 + .../profiling/libkineto/src/EventProfiler.cpp | 635 + .../profiling/libkineto/src/EventProfiler.h | 341 + .../libkineto/src/EventProfilerController.cpp | 423 + .../libkineto/src/EventProfilerController.h | 63 + .../libkineto/src/GenericTraceActivity.cpp | 10 + .../libkineto/src/ILoggerObserver.cpp | 54 + tb_plugins/profiling/libkineto/src/Logger.cpp | 136 + tb_plugins/profiling/libkineto/src/Logger.h | 244 + .../profiling/libkineto/src/LoggerCollector.h | 70 + .../libkineto/src/RoctracerActivityApi.cpp | 569 + .../libkineto/src/RoctracerActivityApi.h | 171 + .../libkineto/src/RoctracerActivityBuffer.h | 30 + .../profiling/libkineto/src/SampleListener.h | 146 + .../profiling/libkineto/src/ScopeExit.h | 29 + .../profiling/libkineto/src/ThreadUtil.cpp | 203 + .../profiling/libkineto/src/WeakSymbols.cpp | 12 + .../profiling/libkineto/src/cupti_call.h | 33 + .../profiling/libkineto/src/cupti_strings.cpp | 502 + .../profiling/libkineto/src/cupti_strings.h | 14 + tb_plugins/profiling/libkineto/src/init.cpp | 139 + .../profiling/libkineto/src/libkineto_api.cpp | 41 + .../profiling/libkineto/src/output_base.h | 104 + .../profiling/libkineto/src/output_csv.cpp | 88 + .../profiling/libkineto/src/output_csv.h | 39 + .../profiling/libkineto/src/output_json.cpp | 583 + .../profiling/libkineto/src/output_json.h | 91 + .../profiling/libkineto/src/output_membuf.h | 130 + .../profiling/libkineto/test/CMakeLists.txt | 3 + .../profiling/libkineto/test/ConfigTest.cpp | 315 + .../test/CuptiActivityProfilerTest.cpp | 629 + .../libkineto/test/CuptiCallbackApiTest.cpp | 239 + .../libkineto/test/CuptiProfilerApiTest.cu | 353 + .../test/CuptiRangeProfilerApiTest.cpp | 113 + .../test/CuptiRangeProfilerConfigTest.cpp | 67 + .../test/CuptiRangeProfilerTestUtil.h | 96 + .../libkineto/test/CuptiStringsTest.cpp | 29 + .../libkineto/test/EventProfilerTest.cpp | 578 + .../libkineto/test/LoggerObserverTest.cpp | 96 + .../test/MockActivitySubProfiler.cpp | 49 + .../libkineto/test/MockActivitySubProfiler.h | 72 + .../profiling/libkineto/test/PidInfoTest.cpp | 27 + tb_plugins/profiling/tb_plugin/.flake8 | 3 + tb_plugins/profiling/tb_plugin/.gitignore | 4 + .../tb_plugin/.pre-commit-config.yaml | 34 + tb_plugins/profiling/tb_plugin/LICENSE | 33 + tb_plugins/profiling/tb_plugin/README.md | 478 + .../tb_plugin/ci_scripts/install_env.sh | 34 + .../tb_plugin/docs/gpu_utilization.md | 22 + .../tb_plugin/docs/images/control_panel.PNG | Bin 0 -> 16255 bytes .../tb_plugin/docs/images/diff_view.png | Bin 0 -> 232656 bytes .../docs/images/distributed_view.PNG | Bin 0 -> 87217 bytes .../tb_plugin/docs/images/kernel_view.PNG | Bin 0 -> 201165 bytes ...kernel_view_group_by_properties_and_op.PNG | Bin 0 -> 77005 bytes .../tb_plugin/docs/images/lightning_view.png | Bin 0 -> 142925 bytes .../tb_plugin/docs/images/memory_view.PNG | Bin 0 -> 128195 bytes .../tb_plugin/docs/images/module_view.png | Bin 0 -> 395978 bytes .../tb_plugin/docs/images/operator_view.PNG | Bin 0 -> 188368 bytes .../operator_view_group_by_inputshape.PNG | Bin 0 -> 63154 bytes .../tb_plugin/docs/images/overall_view.PNG | Bin 0 -> 121355 bytes .../docs/images/time_breakdown_priority.PNG | Bin 0 -> 9063 bytes .../tb_plugin/docs/images/trace_view.PNG | Bin 0 -> 199110 bytes .../images/trace_view_fwd_bwd_correlation.PNG | Bin 0 -> 150070 bytes .../images/trace_view_gpu_utilization.PNG | Bin 0 -> 149099 bytes .../docs/images/trace_view_launch.PNG | Bin 0 -> 455621 bytes .../docs/images/trace_view_one_step.PNG | Bin 0 -> 109373 bytes .../tb_plugin/docs/images/vscode_stack.PNG | Bin 0 -> 440613 bytes .../tb_plugin/examples/datapipe_example.py | 50 + .../examples/resnet50_autograd_api.py | 46 + .../examples/resnet50_ddp_profiler.py | 95 + .../examples/resnet50_profiler_api.py | 52 + tb_plugins/profiling/tb_plugin/fe/.gitignore | 3 + tb_plugins/profiling/tb_plugin/fe/README.md | 18 + tb_plugins/profiling/tb_plugin/fe/index.html | 10 + .../profiling/tb_plugin/fe/package.json | 44 + .../profiling/tb_plugin/fe/prettier.json | 12 + .../tb_plugin/fe/scripts/add_header.py | 32 + .../profiling/tb_plugin/fe/scripts/build.sh | 13 + .../profiling/tb_plugin/fe/scripts/setup.sh | 25 + .../profiling/tb_plugin/fe/src/api/README.md | 13 + .../tb_plugin/fe/src/api/generated/api.ts | 4535 +++++++ .../fe/src/api/generated/configuration.ts | 69 + .../fe/src/api/generated/custom.d.ts | 6 + .../tb_plugin/fe/src/api/generated/index.ts | 19 + .../profiling/tb_plugin/fe/src/api/index.ts | 8 + .../profiling/tb_plugin/fe/src/api/mock.ts | 6716 ++++++++++ .../tb_plugin/fe/src/api/openapi.yaml | 1204 ++ tb_plugins/profiling/tb_plugin/fe/src/app.tsx | 605 + .../fe/src/components/DataLoading.tsx | 19 + .../fe/src/components/DiffOverview.tsx | 855 ++ .../fe/src/components/DistributedView.tsx | 300 + .../src/components/FullCircularProgress.tsx | 23 + .../fe/src/components/GpuInfoTable.tsx | 134 + .../tb_plugin/fe/src/components/Kernel.tsx | 282 + .../fe/src/components/MemoryView.tsx | 465 + .../fe/src/components/ModuleView.tsx | 263 + .../tb_plugin/fe/src/components/Operator.tsx | 308 + .../tb_plugin/fe/src/components/Overview.tsx | 220 + .../fe/src/components/TextListItem.tsx | 89 + .../fe/src/components/TooltipDescriptions.ts | 32 + .../tb_plugin/fe/src/components/TraceView.tsx | 86 + .../src/components/charts/AntTableChart.tsx | 110 + .../fe/src/components/charts/AreaChart.tsx | 70 + .../fe/src/components/charts/ColumnChart.tsx | 87 + .../fe/src/components/charts/LineChart.tsx | 134 + .../fe/src/components/charts/PieChart.tsx | 106 + .../components/charts/SteppedAreaChart.tsx | 75 + .../fe/src/components/charts/TableChart.tsx | 87 + .../tb_plugin/fe/src/components/helpers.tsx | 49 + .../src/components/tables/CallFrameList.tsx | 42 + .../src/components/tables/CallStackTable.tsx | 95 + .../fe/src/components/tables/ExpandIcon.tsx | 34 + .../components/tables/MemoryStatsTable.tsx | 85 + .../src/components/tables/NavToCodeButton.tsx | 29 + .../src/components/tables/OperationTable.tsx | 93 + .../fe/src/components/tables/common.tsx | 143 + .../fe/src/components/tables/transform.ts | 63 + .../tb_plugin/fe/src/components/transform.ts | 82 + .../tb_plugin/fe/src/constants/groupBy.ts | 13 + .../profiling/tb_plugin/fe/src/gstatic.d.ts | 6 + .../profiling/tb_plugin/fe/src/index.tsx | 9 + .../profiling/tb_plugin/fe/src/setup.tsx | 9 + .../profiling/tb_plugin/fe/src/styles.css | 13 + .../tb_plugin/fe/src/utils/binarysearch.ts | 20 + .../tb_plugin/fe/src/utils/debounce.ts | 21 + .../profiling/tb_plugin/fe/src/utils/def.ts | 18 + .../profiling/tb_plugin/fe/src/utils/hooks.ts | 27 + .../profiling/tb_plugin/fe/src/utils/index.ts | 24 + .../tb_plugin/fe/src/utils/resize.ts | 27 + .../tb_plugin/fe/src/utils/search.ts | 66 + .../profiling/tb_plugin/fe/src/utils/top.ts | 50 + .../profiling/tb_plugin/fe/src/utils/type.ts | 9 + .../tb_plugin/fe/src/utils/vscode.ts | 13 + .../profiling/tb_plugin/fe/tsconfig.json | 18 + .../profiling/tb_plugin/fe/update-static.js | 9 + .../profiling/tb_plugin/fe/webpack.config.js | 40 + tb_plugins/profiling/tb_plugin/fe/yarn.lock | 3672 ++++++ .../packaging/torch_tb_profiler/meta.yaml | 39 + .../worker0.1623143089861.pt.trace.json.gz | Bin 0 -> 1159569 bytes .../worker0.1623143566756.pt.trace.json.gz | Bin 0 -> 1160275 bytes .../worker0.1623212756351.pt.trace.json.gz | Bin 0 -> 1095617 bytes .../worker0.1623213129365.pt.trace.json.gz | Bin 0 -> 1091740 bytes tb_plugins/profiling/tb_plugin/setup.py | 105 + .../tb_plugin/test/gpu_metrics_expected.json | 3105 +++++ .../tb_plugin/test/gpu_metrics_input.json | 3105 +++++ .../tb_plugin/test/result_check_file.txt | 10 + .../test/test_compare_with_autograd.py | 301 + .../profiling/tb_plugin/test/test_diffrun.py | 51 + .../profiling/tb_plugin/test/test_profiler.py | 2752 +++++ .../profiling/tb_plugin/test/test_ranges.py | 50 + .../test/test_tensorboard_end2end.py | 170 + .../tb_plugin/torch_tb_profiler/__init__.py | 7 + .../tb_plugin/torch_tb_profiler/consts.py | 74 + .../torch_tb_profiler/io/__init__.py | 4 + .../torch_tb_profiler/io/azureblob.py | 187 + .../tb_plugin/torch_tb_profiler/io/base.py | 112 + .../tb_plugin/torch_tb_profiler/io/cache.py | 81 + .../tb_plugin/torch_tb_profiler/io/file.py | 622 + .../tb_plugin/torch_tb_profiler/io/gs.py | 127 + .../tb_plugin/torch_tb_profiler/io/utils.py | 72 + .../torch_tb_profiler/multiprocessing.py | 13 + .../tb_plugin/torch_tb_profiler/plugin.py | 557 + .../torch_tb_profiler/profiler/__init__.py | 7 + .../profiler/communication.py | 91 + .../torch_tb_profiler/profiler/data.py | 356 + .../profiler/diffrun/__init__.py | 3 + .../profiler/diffrun/contract.py | 99 + .../profiler/diffrun/operator.py | 124 + .../profiler/diffrun/tree.py | 165 + .../profiler/event_parser.py | 474 + .../profiler/gpu_metrics_parser.py | 314 + .../profiler/kernel_parser.py | 45 + .../torch_tb_profiler/profiler/loader.py | 166 + .../profiler/memory_parser.py | 328 + .../torch_tb_profiler/profiler/module_op.py | 269 + .../torch_tb_profiler/profiler/node.py | 316 + .../torch_tb_profiler/profiler/op_agg.py | 162 + .../torch_tb_profiler/profiler/op_tree.py | 351 + .../profiler/overall_parser.py | 110 + .../torch_tb_profiler/profiler/range_utils.py | 190 + .../profiler/run_generator.py | 572 + .../torch_tb_profiler/profiler/tensor_core.py | 52 + .../profiler/tensor_cores_parser.py | 77 + .../torch_tb_profiler/profiler/trace.py | 231 + .../tb_plugin/torch_tb_profiler/run.py | 485 + .../torch_tb_profiler/static/index.html | 2 + .../torch_tb_profiler/static/index.js | 3 + .../static/trace_embedding.html | 104 + .../static/trace_viewer_full.html | 10174 ++++++++++++++++ .../tb_plugin/torch_tb_profiler/utils.py | 122 + 257 files changed, 65706 insertions(+) create mode 100644 tb_plugins/profiling/.github/workflows/libkineto_ci.yml create mode 100644 tb_plugins/profiling/.github/workflows/tb_plugin_build_pip_package.yml create mode 100644 tb_plugins/profiling/.github/workflows/tb_plugin_ci.yml create mode 100644 tb_plugins/profiling/.gitignore create mode 100644 tb_plugins/profiling/.gitmodules create mode 100644 tb_plugins/profiling/CODE_OF_CONDUCT.md create mode 100644 tb_plugins/profiling/CONTRIBUTING.md create mode 100644 tb_plugins/profiling/LICENSE create mode 100644 tb_plugins/profiling/README.md create mode 100644 tb_plugins/profiling/libkineto/CMakeLists.txt create mode 100644 tb_plugins/profiling/libkineto/README.md create mode 100644 tb_plugins/profiling/libkineto/include/AbstractConfig.h create mode 100644 tb_plugins/profiling/libkineto/include/ActivityProfilerInterface.h create mode 100644 tb_plugins/profiling/libkineto/include/ActivityTraceInterface.h create mode 100644 tb_plugins/profiling/libkineto/include/ActivityType.h create mode 100644 tb_plugins/profiling/libkineto/include/ClientInterface.h create mode 100644 tb_plugins/profiling/libkineto/include/Config.h create mode 100644 tb_plugins/profiling/libkineto/include/GenericTraceActivity.h create mode 100644 tb_plugins/profiling/libkineto/include/IActivityProfiler.h create mode 100644 tb_plugins/profiling/libkineto/include/ILoggerObserver.h create mode 100644 tb_plugins/profiling/libkineto/include/ITraceActivity.h create mode 100644 tb_plugins/profiling/libkineto/include/ThreadUtil.h create mode 100644 tb_plugins/profiling/libkineto/include/TraceSpan.h create mode 100644 tb_plugins/profiling/libkineto/include/libkineto.h create mode 100644 tb_plugins/profiling/libkineto/include/time_since_epoch.h create mode 100644 tb_plugins/profiling/libkineto/libkineto_defs.bzl create mode 100644 tb_plugins/profiling/libkineto/sample_programs/kineto_playground.cpp create mode 100644 tb_plugins/profiling/libkineto/sample_programs/kineto_playground.cu create mode 100644 tb_plugins/profiling/libkineto/sample_programs/kineto_playground.cuh create mode 100644 tb_plugins/profiling/libkineto/src/AbstractConfig.cpp create mode 100644 tb_plugins/profiling/libkineto/src/ActivityBuffers.h create mode 100644 tb_plugins/profiling/libkineto/src/ActivityLoggerFactory.h create mode 100644 tb_plugins/profiling/libkineto/src/ActivityProfilerController.cpp create mode 100644 tb_plugins/profiling/libkineto/src/ActivityProfilerController.h create mode 100644 tb_plugins/profiling/libkineto/src/ActivityProfilerProxy.cpp create mode 100644 tb_plugins/profiling/libkineto/src/ActivityProfilerProxy.h create mode 100644 tb_plugins/profiling/libkineto/src/ActivityTrace.h create mode 100644 tb_plugins/profiling/libkineto/src/ActivityType.cpp create mode 100644 tb_plugins/profiling/libkineto/src/Config.cpp create mode 100644 tb_plugins/profiling/libkineto/src/ConfigLoader.cpp create mode 100644 tb_plugins/profiling/libkineto/src/ConfigLoader.h create mode 100644 tb_plugins/profiling/libkineto/src/CudaDeviceProperties.cpp create mode 100644 tb_plugins/profiling/libkineto/src/CudaDeviceProperties.h create mode 100644 tb_plugins/profiling/libkineto/src/CuptiActivity.h create mode 100644 tb_plugins/profiling/libkineto/src/CuptiActivity.tpp create mode 100644 tb_plugins/profiling/libkineto/src/CuptiActivityApi.cpp create mode 100644 tb_plugins/profiling/libkineto/src/CuptiActivityApi.h create mode 100644 tb_plugins/profiling/libkineto/src/CuptiActivityBuffer.h create mode 100644 tb_plugins/profiling/libkineto/src/CuptiActivityPlatform.cpp create mode 100644 tb_plugins/profiling/libkineto/src/CuptiActivityPlatform.h create mode 100644 tb_plugins/profiling/libkineto/src/CuptiActivityProfiler.cpp create mode 100644 tb_plugins/profiling/libkineto/src/CuptiActivityProfiler.h create mode 100644 tb_plugins/profiling/libkineto/src/CuptiCallbackApi.cpp create mode 100644 tb_plugins/profiling/libkineto/src/CuptiCallbackApi.h create mode 100644 tb_plugins/profiling/libkineto/src/CuptiCallbackApiMock.h create mode 100644 tb_plugins/profiling/libkineto/src/CuptiEventApi.cpp create mode 100644 tb_plugins/profiling/libkineto/src/CuptiEventApi.h create mode 100644 tb_plugins/profiling/libkineto/src/CuptiMetricApi.cpp create mode 100644 tb_plugins/profiling/libkineto/src/CuptiMetricApi.h create mode 100644 tb_plugins/profiling/libkineto/src/CuptiNvPerfMetric.cpp create mode 100644 tb_plugins/profiling/libkineto/src/CuptiNvPerfMetric.h create mode 100644 tb_plugins/profiling/libkineto/src/CuptiRangeProfilerApi.cpp create mode 100644 tb_plugins/profiling/libkineto/src/CuptiRangeProfilerApi.h create mode 100644 tb_plugins/profiling/libkineto/src/CuptiRangeProfilerConfig.cpp create mode 100644 tb_plugins/profiling/libkineto/src/CuptiRangeProfilerConfig.h create mode 100644 tb_plugins/profiling/libkineto/src/DaemonConfigLoader.h create mode 100644 tb_plugins/profiling/libkineto/src/Demangle.cpp create mode 100644 tb_plugins/profiling/libkineto/src/Demangle.h create mode 100644 tb_plugins/profiling/libkineto/src/EventProfiler.cpp create mode 100644 tb_plugins/profiling/libkineto/src/EventProfiler.h create mode 100644 tb_plugins/profiling/libkineto/src/EventProfilerController.cpp create mode 100644 tb_plugins/profiling/libkineto/src/EventProfilerController.h create mode 100644 tb_plugins/profiling/libkineto/src/GenericTraceActivity.cpp create mode 100644 tb_plugins/profiling/libkineto/src/ILoggerObserver.cpp create mode 100644 tb_plugins/profiling/libkineto/src/Logger.cpp create mode 100644 tb_plugins/profiling/libkineto/src/Logger.h create mode 100644 tb_plugins/profiling/libkineto/src/LoggerCollector.h create mode 100644 tb_plugins/profiling/libkineto/src/RoctracerActivityApi.cpp create mode 100644 tb_plugins/profiling/libkineto/src/RoctracerActivityApi.h create mode 100644 tb_plugins/profiling/libkineto/src/RoctracerActivityBuffer.h create mode 100644 tb_plugins/profiling/libkineto/src/SampleListener.h create mode 100644 tb_plugins/profiling/libkineto/src/ScopeExit.h create mode 100644 tb_plugins/profiling/libkineto/src/ThreadUtil.cpp create mode 100644 tb_plugins/profiling/libkineto/src/WeakSymbols.cpp create mode 100644 tb_plugins/profiling/libkineto/src/cupti_call.h create mode 100644 tb_plugins/profiling/libkineto/src/cupti_strings.cpp create mode 100644 tb_plugins/profiling/libkineto/src/cupti_strings.h create mode 100644 tb_plugins/profiling/libkineto/src/init.cpp create mode 100644 tb_plugins/profiling/libkineto/src/libkineto_api.cpp create mode 100644 tb_plugins/profiling/libkineto/src/output_base.h create mode 100644 tb_plugins/profiling/libkineto/src/output_csv.cpp create mode 100644 tb_plugins/profiling/libkineto/src/output_csv.h create mode 100644 tb_plugins/profiling/libkineto/src/output_json.cpp create mode 100644 tb_plugins/profiling/libkineto/src/output_json.h create mode 100644 tb_plugins/profiling/libkineto/src/output_membuf.h create mode 100644 tb_plugins/profiling/libkineto/test/CMakeLists.txt create mode 100644 tb_plugins/profiling/libkineto/test/ConfigTest.cpp create mode 100644 tb_plugins/profiling/libkineto/test/CuptiActivityProfilerTest.cpp create mode 100644 tb_plugins/profiling/libkineto/test/CuptiCallbackApiTest.cpp create mode 100644 tb_plugins/profiling/libkineto/test/CuptiProfilerApiTest.cu create mode 100644 tb_plugins/profiling/libkineto/test/CuptiRangeProfilerApiTest.cpp create mode 100644 tb_plugins/profiling/libkineto/test/CuptiRangeProfilerConfigTest.cpp create mode 100644 tb_plugins/profiling/libkineto/test/CuptiRangeProfilerTestUtil.h create mode 100644 tb_plugins/profiling/libkineto/test/CuptiStringsTest.cpp create mode 100644 tb_plugins/profiling/libkineto/test/EventProfilerTest.cpp create mode 100644 tb_plugins/profiling/libkineto/test/LoggerObserverTest.cpp create mode 100644 tb_plugins/profiling/libkineto/test/MockActivitySubProfiler.cpp create mode 100644 tb_plugins/profiling/libkineto/test/MockActivitySubProfiler.h create mode 100644 tb_plugins/profiling/libkineto/test/PidInfoTest.cpp create mode 100644 tb_plugins/profiling/tb_plugin/.flake8 create mode 100644 tb_plugins/profiling/tb_plugin/.gitignore create mode 100644 tb_plugins/profiling/tb_plugin/.pre-commit-config.yaml create mode 100644 tb_plugins/profiling/tb_plugin/LICENSE create mode 100644 tb_plugins/profiling/tb_plugin/README.md create mode 100644 tb_plugins/profiling/tb_plugin/ci_scripts/install_env.sh create mode 100644 tb_plugins/profiling/tb_plugin/docs/gpu_utilization.md create mode 100644 tb_plugins/profiling/tb_plugin/docs/images/control_panel.PNG create mode 100644 tb_plugins/profiling/tb_plugin/docs/images/diff_view.png create mode 100644 tb_plugins/profiling/tb_plugin/docs/images/distributed_view.PNG create mode 100644 tb_plugins/profiling/tb_plugin/docs/images/kernel_view.PNG create mode 100644 tb_plugins/profiling/tb_plugin/docs/images/kernel_view_group_by_properties_and_op.PNG create mode 100644 tb_plugins/profiling/tb_plugin/docs/images/lightning_view.png create mode 100644 tb_plugins/profiling/tb_plugin/docs/images/memory_view.PNG create mode 100644 tb_plugins/profiling/tb_plugin/docs/images/module_view.png create mode 100644 tb_plugins/profiling/tb_plugin/docs/images/operator_view.PNG create mode 100644 tb_plugins/profiling/tb_plugin/docs/images/operator_view_group_by_inputshape.PNG create mode 100644 tb_plugins/profiling/tb_plugin/docs/images/overall_view.PNG create mode 100644 tb_plugins/profiling/tb_plugin/docs/images/time_breakdown_priority.PNG create mode 100644 tb_plugins/profiling/tb_plugin/docs/images/trace_view.PNG create mode 100644 tb_plugins/profiling/tb_plugin/docs/images/trace_view_fwd_bwd_correlation.PNG create mode 100644 tb_plugins/profiling/tb_plugin/docs/images/trace_view_gpu_utilization.PNG create mode 100644 tb_plugins/profiling/tb_plugin/docs/images/trace_view_launch.PNG create mode 100644 tb_plugins/profiling/tb_plugin/docs/images/trace_view_one_step.PNG create mode 100644 tb_plugins/profiling/tb_plugin/docs/images/vscode_stack.PNG create mode 100644 tb_plugins/profiling/tb_plugin/examples/datapipe_example.py create mode 100644 tb_plugins/profiling/tb_plugin/examples/resnet50_autograd_api.py create mode 100644 tb_plugins/profiling/tb_plugin/examples/resnet50_ddp_profiler.py create mode 100644 tb_plugins/profiling/tb_plugin/examples/resnet50_profiler_api.py create mode 100644 tb_plugins/profiling/tb_plugin/fe/.gitignore create mode 100644 tb_plugins/profiling/tb_plugin/fe/README.md create mode 100644 tb_plugins/profiling/tb_plugin/fe/index.html create mode 100644 tb_plugins/profiling/tb_plugin/fe/package.json create mode 100644 tb_plugins/profiling/tb_plugin/fe/prettier.json create mode 100644 tb_plugins/profiling/tb_plugin/fe/scripts/add_header.py create mode 100644 tb_plugins/profiling/tb_plugin/fe/scripts/build.sh create mode 100644 tb_plugins/profiling/tb_plugin/fe/scripts/setup.sh create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/api/README.md create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/api/generated/api.ts create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/api/generated/configuration.ts create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/api/generated/custom.d.ts create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/api/generated/index.ts create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/api/index.ts create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/api/mock.ts create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/api/openapi.yaml create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/app.tsx create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/components/DataLoading.tsx create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/components/DiffOverview.tsx create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/components/DistributedView.tsx create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/components/FullCircularProgress.tsx create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/components/GpuInfoTable.tsx create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/components/Kernel.tsx create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/components/MemoryView.tsx create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/components/ModuleView.tsx create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/components/Operator.tsx create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/components/Overview.tsx create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/components/TextListItem.tsx create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/components/TooltipDescriptions.ts create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/components/TraceView.tsx create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/components/charts/AntTableChart.tsx create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/components/charts/AreaChart.tsx create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/components/charts/ColumnChart.tsx create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/components/charts/LineChart.tsx create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/components/charts/PieChart.tsx create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/components/charts/SteppedAreaChart.tsx create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/components/charts/TableChart.tsx create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/components/helpers.tsx create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/components/tables/CallFrameList.tsx create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/components/tables/CallStackTable.tsx create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/components/tables/ExpandIcon.tsx create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/components/tables/MemoryStatsTable.tsx create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/components/tables/NavToCodeButton.tsx create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/components/tables/OperationTable.tsx create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/components/tables/common.tsx create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/components/tables/transform.ts create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/components/transform.ts create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/constants/groupBy.ts create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/gstatic.d.ts create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/index.tsx create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/setup.tsx create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/styles.css create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/utils/binarysearch.ts create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/utils/debounce.ts create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/utils/def.ts create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/utils/hooks.ts create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/utils/index.ts create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/utils/resize.ts create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/utils/search.ts create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/utils/top.ts create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/utils/type.ts create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/utils/vscode.ts create mode 100644 tb_plugins/profiling/tb_plugin/fe/tsconfig.json create mode 100644 tb_plugins/profiling/tb_plugin/fe/update-static.js create mode 100644 tb_plugins/profiling/tb_plugin/fe/webpack.config.js create mode 100644 tb_plugins/profiling/tb_plugin/fe/yarn.lock create mode 100644 tb_plugins/profiling/tb_plugin/packaging/torch_tb_profiler/meta.yaml create mode 100644 tb_plugins/profiling/tb_plugin/samples/resnet50_num_workers_0/worker0.1623143089861.pt.trace.json.gz create mode 100644 tb_plugins/profiling/tb_plugin/samples/resnet50_num_workers_0/worker0.1623143566756.pt.trace.json.gz create mode 100644 tb_plugins/profiling/tb_plugin/samples/resnet50_num_workers_4/worker0.1623212756351.pt.trace.json.gz create mode 100644 tb_plugins/profiling/tb_plugin/samples/resnet50_num_workers_4/worker0.1623213129365.pt.trace.json.gz create mode 100644 tb_plugins/profiling/tb_plugin/setup.py create mode 100644 tb_plugins/profiling/tb_plugin/test/gpu_metrics_expected.json create mode 100644 tb_plugins/profiling/tb_plugin/test/gpu_metrics_input.json create mode 100644 tb_plugins/profiling/tb_plugin/test/result_check_file.txt create mode 100644 tb_plugins/profiling/tb_plugin/test/test_compare_with_autograd.py create mode 100644 tb_plugins/profiling/tb_plugin/test/test_diffrun.py create mode 100644 tb_plugins/profiling/tb_plugin/test/test_profiler.py create mode 100644 tb_plugins/profiling/tb_plugin/test/test_ranges.py create mode 100644 tb_plugins/profiling/tb_plugin/test/test_tensorboard_end2end.py create mode 100644 tb_plugins/profiling/tb_plugin/torch_tb_profiler/__init__.py create mode 100644 tb_plugins/profiling/tb_plugin/torch_tb_profiler/consts.py create mode 100644 tb_plugins/profiling/tb_plugin/torch_tb_profiler/io/__init__.py create mode 100644 tb_plugins/profiling/tb_plugin/torch_tb_profiler/io/azureblob.py create mode 100644 tb_plugins/profiling/tb_plugin/torch_tb_profiler/io/base.py create mode 100644 tb_plugins/profiling/tb_plugin/torch_tb_profiler/io/cache.py create mode 100644 tb_plugins/profiling/tb_plugin/torch_tb_profiler/io/file.py create mode 100644 tb_plugins/profiling/tb_plugin/torch_tb_profiler/io/gs.py create mode 100644 tb_plugins/profiling/tb_plugin/torch_tb_profiler/io/utils.py create mode 100644 tb_plugins/profiling/tb_plugin/torch_tb_profiler/multiprocessing.py create mode 100644 tb_plugins/profiling/tb_plugin/torch_tb_profiler/plugin.py create mode 100644 tb_plugins/profiling/tb_plugin/torch_tb_profiler/profiler/__init__.py create mode 100644 tb_plugins/profiling/tb_plugin/torch_tb_profiler/profiler/communication.py create mode 100644 tb_plugins/profiling/tb_plugin/torch_tb_profiler/profiler/data.py create mode 100644 tb_plugins/profiling/tb_plugin/torch_tb_profiler/profiler/diffrun/__init__.py create mode 100644 tb_plugins/profiling/tb_plugin/torch_tb_profiler/profiler/diffrun/contract.py create mode 100644 tb_plugins/profiling/tb_plugin/torch_tb_profiler/profiler/diffrun/operator.py create mode 100644 tb_plugins/profiling/tb_plugin/torch_tb_profiler/profiler/diffrun/tree.py create mode 100644 tb_plugins/profiling/tb_plugin/torch_tb_profiler/profiler/event_parser.py create mode 100644 tb_plugins/profiling/tb_plugin/torch_tb_profiler/profiler/gpu_metrics_parser.py create mode 100644 tb_plugins/profiling/tb_plugin/torch_tb_profiler/profiler/kernel_parser.py create mode 100644 tb_plugins/profiling/tb_plugin/torch_tb_profiler/profiler/loader.py create mode 100644 tb_plugins/profiling/tb_plugin/torch_tb_profiler/profiler/memory_parser.py create mode 100644 tb_plugins/profiling/tb_plugin/torch_tb_profiler/profiler/module_op.py create mode 100644 tb_plugins/profiling/tb_plugin/torch_tb_profiler/profiler/node.py create mode 100644 tb_plugins/profiling/tb_plugin/torch_tb_profiler/profiler/op_agg.py create mode 100644 tb_plugins/profiling/tb_plugin/torch_tb_profiler/profiler/op_tree.py create mode 100644 tb_plugins/profiling/tb_plugin/torch_tb_profiler/profiler/overall_parser.py create mode 100644 tb_plugins/profiling/tb_plugin/torch_tb_profiler/profiler/range_utils.py create mode 100644 tb_plugins/profiling/tb_plugin/torch_tb_profiler/profiler/run_generator.py create mode 100644 tb_plugins/profiling/tb_plugin/torch_tb_profiler/profiler/tensor_core.py create mode 100644 tb_plugins/profiling/tb_plugin/torch_tb_profiler/profiler/tensor_cores_parser.py create mode 100644 tb_plugins/profiling/tb_plugin/torch_tb_profiler/profiler/trace.py create mode 100644 tb_plugins/profiling/tb_plugin/torch_tb_profiler/run.py create mode 100644 tb_plugins/profiling/tb_plugin/torch_tb_profiler/static/index.html create mode 100644 tb_plugins/profiling/tb_plugin/torch_tb_profiler/static/index.js create mode 100644 tb_plugins/profiling/tb_plugin/torch_tb_profiler/static/trace_embedding.html create mode 100644 tb_plugins/profiling/tb_plugin/torch_tb_profiler/static/trace_viewer_full.html create mode 100644 tb_plugins/profiling/tb_plugin/torch_tb_profiler/utils.py diff --git a/tb_plugins/profiling/.github/workflows/libkineto_ci.yml b/tb_plugins/profiling/.github/workflows/libkineto_ci.yml new file mode 100644 index 000000000..3133d6400 --- /dev/null +++ b/tb_plugins/profiling/.github/workflows/libkineto_ci.yml @@ -0,0 +1,56 @@ +name: LIBKINETOCI + +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + build: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + + steps: + - uses: actions/checkout@v2 + - name: Checkout submodules + shell: bash + run: | + auth_header="$(git config --local --get http.https://github.com/.extraheader)" + git submodule sync --recursive + git -c "http.extraheader=$auth_header" -c protocol.version=2 submodule update --init --force --recursive --depth=1 + + - name: Get env vars + run: | + echo GITHUB_WORKFLOW = $GITHUB_WORKFLOW + echo HOME = $HOME + echo GITHUB_ACTION = $GITHUB_ACTION + echo GITHUB_ACTIONS = $GITHUB_ACTIONS + echo GITHUB_REPOSITORY = $GITHUB_REPOSITORY + echo GITHUB_EVENT_NAME = $GITHUB_EVENT_NAME + echo GITHUB_EVENT_PATH = $GITHUB_EVENT_PATH + echo GITHUB_WORKSPACE = $GITHUB_WORKSPACE + echo GITHUB_SHA = $GITHUB_SHA + echo GITHUB_REF = $GITHUB_REF + c++ --verbose + + # TODO: Figure out how to install cupti headers T84637671 + - name: Build static lib + run: | + set -e + mkdir build_static + cd build_static + cmake -DKINETO_LIBRARY_TYPE=static ../libkineto/ + make -j + + - name: Build shared lib + run: | + set -e + mkdir build_shared + cd build_shared + cmake -DKINETO_LIBRARY_TYPE=shared ../libkineto/ + make -j diff --git a/tb_plugins/profiling/.github/workflows/tb_plugin_build_pip_package.yml b/tb_plugins/profiling/.github/workflows/tb_plugin_build_pip_package.yml new file mode 100644 index 000000000..9bdafcc44 --- /dev/null +++ b/tb_plugins/profiling/.github/workflows/tb_plugin_build_pip_package.yml @@ -0,0 +1,19 @@ +name: Build torch-tb-profiler Pip Package + +on: + # TODO: Add an on_release trigger to build on tags + workflow_dispatch: + +jobs: + build-package: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: build pip package + run: | + set -e + cd tb_plugin + python setup.py sdist bdist_wheel + cd dist/ + pip install *.whl + python -c "import torch_tb_profiler;print(torch_tb_profiler.__version__)" diff --git a/tb_plugins/profiling/.github/workflows/tb_plugin_ci.yml b/tb_plugins/profiling/.github/workflows/tb_plugin_ci.yml new file mode 100644 index 000000000..1b59a7bf9 --- /dev/null +++ b/tb_plugins/profiling/.github/workflows/tb_plugin_ci.yml @@ -0,0 +1,57 @@ +name: TB_Plugin_CI + +on: + push: + branches: + - main + - release/** + - plugin/** + + pull_request: + branches: + - main + - release/** + - plugin/** + +jobs: + generate-matrix: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - id: set-matrix + run: | + echo $GITHUB_BASE_REF + if [ $GITHUB_BASE_REF == "plugin/vnext" ] + then + echo "::set-output name=matrix::{\"python-version\":[3.7, 3.8, 3.9], \"cuda-version\":[\"cpu\"], \"pytorch-version\":[\"nightly\"]}" + else + echo "::set-output name=matrix::{\"python-version\":[3.7, 3.8, 3.9], \"cuda-version\":[\"cpu\"], \"pytorch-version\":[\"nightly\", \"1.11rc\", \"stable\"]}" + fi + + build: + needs: generate-matrix + runs-on: ubuntu-latest + strategy: + matrix: ${{fromJSON(needs.generate-matrix.outputs.matrix)}} + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + architecture: 'x64' + - name: Test + env: + CUDA_VERSION: ${{ matrix.cuda-version }} + PYTORCH_VERSION: ${{ matrix.pytorch-version }} + TORCH_PROFILER_LOG_LEVEL: DEBUG + GRPC_VERBOSITY: DEBUG + GRPC_ENABLE_FORK_SUPPORT: 'False' + run: | + set -e + cd tb_plugin + sh ./ci_scripts/install_env.sh + pip install .[gs] + cd test + pytest diff --git a/tb_plugins/profiling/.gitignore b/tb_plugins/profiling/.gitignore new file mode 100644 index 000000000..ce186381c --- /dev/null +++ b/tb_plugins/profiling/.gitignore @@ -0,0 +1,3 @@ +# ignore common items +.idea +.vscode diff --git a/tb_plugins/profiling/.gitmodules b/tb_plugins/profiling/.gitmodules new file mode 100644 index 000000000..4660ee8bc --- /dev/null +++ b/tb_plugins/profiling/.gitmodules @@ -0,0 +1,6 @@ +[submodule "libkineto/third_party/googletest"] + path = libkineto/third_party/googletest + url = https://github.com/google/googletest.git +[submodule "libkineto/third_party/fmt"] + path = libkineto/third_party/fmt + url = https://github.com/fmtlib/fmt.git diff --git a/tb_plugins/profiling/CODE_OF_CONDUCT.md b/tb_plugins/profiling/CODE_OF_CONDUCT.md new file mode 100644 index 000000000..a0cbeaab7 --- /dev/null +++ b/tb_plugins/profiling/CODE_OF_CONDUCT.md @@ -0,0 +1,77 @@ +# Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to make participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, sex characteristics, gender identity and expression, +level of experience, education, socio-economic status, nationality, personal +appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies within all project spaces, and it also applies when +an individual is representing the project or its community in public spaces. +Examples of representing a project or community include using an official +project e-mail address, posting via an official social media account, or acting +as an appointed representative at an online or offline event. Representation of +a project may be further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at . All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see +https://www.contributor-covenant.org/faq + diff --git a/tb_plugins/profiling/CONTRIBUTING.md b/tb_plugins/profiling/CONTRIBUTING.md new file mode 100644 index 000000000..a2e931bb6 --- /dev/null +++ b/tb_plugins/profiling/CONTRIBUTING.md @@ -0,0 +1,34 @@ +# Contributing to Kineto +We want to make contributing to this project as easy and transparent as +possible. + +## Code of Conduct +The code of conduct is described in [`CODE_OF_CONDUCT.md`](CODE_OF_CONDUCT.md). + +## Pull Requests +We actively welcome your pull requests. + +1. Fork the repo and create your branch from `main`. +2. If you've added code that should be tested, add tests. +3. If you've changed APIs, update the documentation. +4. Ensure the test suite passes. +5. Make sure your code lints. +6. If you haven't already, complete the Contributor License Agreement ("CLA"). + +## Contributor License Agreement ("CLA") +In order to accept your pull request, we need you to submit a CLA. You only need +to do this once to work on any of Facebook's open source projects. + +Complete your CLA here: + +## Issues +We use GitHub issues to track public bugs. Please ensure your description is +clear and has sufficient instructions to be able to reproduce the issue. + +Facebook has a [bounty program](https://www.facebook.com/whitehat/) for the safe +disclosure of security bugs. In those cases, please go through the process +outlined on that page and do not file a public issue. + +## License +By contributing to Kineto, you agree that your contributions will be licensed +under the LICENSE file in the root directory of this source tree. diff --git a/tb_plugins/profiling/LICENSE b/tb_plugins/profiling/LICENSE new file mode 100644 index 000000000..edb179715 --- /dev/null +++ b/tb_plugins/profiling/LICENSE @@ -0,0 +1,33 @@ +BSD License + +For Kineto software + +Copyright (c) Facebook, Inc. and its affiliates. All rights reserved. + +All contributions by Microsoft: +Copyright (c) Microsoft Corporation. (The Azure AI Platform team) + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Facebook nor the names of its contributors may be used to + endorse or promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/tb_plugins/profiling/README.md b/tb_plugins/profiling/README.md new file mode 100644 index 000000000..0353b1b4d --- /dev/null +++ b/tb_plugins/profiling/README.md @@ -0,0 +1,38 @@ +# Kineto + +Kineto is part of the PyTorch Profiler. + +The Kineto project was started to help enable +- **performance observability and diagnostics** across common ML bottleneck components +- **actionable recommendations** for common issues +- integration of external system-level profiling tools +- integration with popular visualization platforms and analysis pipelines + +A central component is libkineto, a profiling library with special focus on low-overhead GPU timeline tracing. + +The PyTorch Profiler TensorBoard plugin provides powerful and intuitive visualizations of profiling results, as well as actionable recommendations, and is the best way to experience the new PyTorch Profiler. + +## Libkineto +Libkineto is an in-process profiling library integrated with the PyTorch Profiler. Please refer to the [README](libkineto/README.md) file in the `libkineto` folder as well as documentation on the [new PyTorch Profiler API](https://pytorch.org/docs/master/profiler.html). + +## PyTorch TensorBoard Profiler +The goal of the PyTorch TensorBoard Profiler is to provide a seamless and intuitive end-to-end profiling experience, including straightforward collection from PyTorch and insightful visualizations and recommendations in the TensorBoard UI. +Please refer to the [README](tb_plugin/README.md) file in the `tb_plugin` folder. + +## Future Development Direction: +Some areas we're currently working on: +- Support for tracing distributed workloads +- Trace processing, analysis and recommendation engine +- System-level activities, multiple tracing sources +- Profiling and monitoring daemon for larger scale deployments + +## Releases and Contributing +We will follow the PyTorch release schedule which roughly happens on a 3 month basis. + +We appreciate all contributions. If you are planning to contribute back bug-fixes, please do so without any further discussion. + +If you plan to contribute new features, please first open an issue and discuss the feature with us. Sending a PR without discussion might end up resulting in a rejected PR because we might be taking the infrastructure in a different direction than you might be aware of. We expect the architecture to keep evolving. + +## License +Kineto has a BSD-style license, as found in the [LICENSE](LICENSE) file. + diff --git a/tb_plugins/profiling/libkineto/CMakeLists.txt b/tb_plugins/profiling/libkineto/CMakeLists.txt new file mode 100644 index 000000000..63966de80 --- /dev/null +++ b/tb_plugins/profiling/libkineto/CMakeLists.txt @@ -0,0 +1,198 @@ +cmake_minimum_required(VERSION 3.5 FATAL_ERROR) + +list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules") + +#install libraries into correct locations on all platforms +include(GNUInstallDirs) + +# function to extract filelists from libkineto_defs.bzl file +find_package(PythonInterp) +function(get_filelist name outputvar) + execute_process( + COMMAND "${PYTHON_EXECUTABLE}" -c + "exec(open('libkineto_defs.bzl').read());print(';'.join(${name}))" + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + OUTPUT_VARIABLE _tempvar) + string(REPLACE "\n" "" _tempvar "${_tempvar}") + set(${outputvar} ${_tempvar} PARENT_SCOPE) +endfunction() + +project(kineto VERSION 0.1 LANGUAGES CXX C) + +set(KINETO_LIBRARY_TYPE "default" CACHE STRING + "Type of library (default, static or shared) to build") +set_property(CACHE KINETO_LIBRARY_TYPE PROPERTY STRINGS default shared) +option(KINETO_BUILD_TESTS "Build kineto unit tests" ON) + +set(LIBKINETO_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/src") +set(LIBKINETO_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/include") +set(LIBKINETO_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}) +set(LIBKINETO_THIRDPARTY_DIR "${CMAKE_CURRENT_SOURCE_DIR}/third_party") +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + +#We should default to a Release build +if (NOT CMAKE_BUILD_TYPE OR CMAKE_BUILD_TYPE STREQUAL "") + set(CMAKE_BUILD_TYPE "Release" CACHE STRING "" FORCE) +endif() + +if (NOT CUDA_SOURCE_DIR) + set(CUDA_SOURCE_DIR "$ENV{CUDA_SOURCE_DIR}") + message(INFO " CUDA_SOURCE_DIR = ${CUDA_SOURCE_DIR}") +endif() + +if (NOT ROCM_SOURCE_DIR) + set(ROCM_SOURCE_DIR "$ENV{ROCM_SOURCE_DIR}") + message(INFO " ROCM_SOURCE_DIR = ${ROCM_SOURCE_DIR}") +endif() + +# Set LIBKINETO_NOCUPTI to explicitly disable CUPTI +# Otherwise, CUPTI is disabled if not found +IF (NOT CUDA_SOURCE_DIR OR NOT CUPTI_INCLUDE_DIR OR NOT CUDA_cupti_LIBRARY) + set(LIBKINETO_NOCUPTI ON CACHE BOOL "" FORCE) +endif() + +IF (NOT ROCM_SOURCE_DIR AND NOT ROCTRACER_INCLUDE_DIR) + set(LIBKINETO_NOROCTRACER ON CACHE BOOL "" FORCE) +endif() + +# Define file lists +if (LIBKINETO_NOCUPTI AND LIBKINETO_NOROCTRACER) + get_filelist("get_libkineto_cpu_only_srcs(with_api=False)" LIBKINETO_SRCS) + message(INFO " CUPTI unavailable or disabled - not building GPU profilers") +elseif(NOT LIBKINETO_NOROCTRACER) + get_filelist("get_libkineto_roctracer_srcs()" LIBKINETO_SRCS) + message(INFO " Building with roctracer") +else() + get_filelist("get_libkineto_cupti_srcs(with_api=False)" LIBKINETO_SRCS) +endif() +get_filelist("get_libkineto_public_headers()" LIBKINETO_PUBLIC_HEADERS) +get_filelist("get_libkineto_api_srcs()" LIBKINETO_API_SRCS) + +add_library(kineto_base OBJECT ${LIBKINETO_SRCS}) +add_library(kineto_api OBJECT ${LIBKINETO_API_SRCS}) + +# Make libraries depend on libkineto_defs.bzl +add_custom_target(libkineto_defs.bzl DEPENDS libkineto_defs.bzl) +add_dependencies(kineto_base libkineto_defs.bzl) + +set_target_properties(kineto_base kineto_api PROPERTIES + CXX_STANDARD 14 + CXX_STANDARD_REQUIRED YES + CXX_EXTENSIONS NO + CXX_VISIBILITY_PRESET hidden) + +set(KINETO_COMPILE_OPTIONS "-DKINETO_NAMESPACE=libkineto") +list(APPEND KINETO_COMPILE_OPTIONS "-DFMT_HEADER_ONLY") +if(NOT MSVC) + list(APPEND KINETO_COMPILE_OPTIONS "-std=c++14") +else() + list(APPEND KINETO_COMPILE_OPTIONS "/std:c++14") + list(APPEND KINETO_COMPILE_OPTIONS "-DWIN32_LEAN_AND_MEAN") + list(APPEND KINETO_COMPILE_OPTIONS "-DNOGDI") +endif() +if (NOT LIBKINETO_NOCUPTI) + list(APPEND KINETO_COMPILE_OPTIONS "-DHAS_CUPTI") +endif() +if (NOT LIBKINETO_NOROCTRACER) + target_compile_options(kineto_base PRIVATE "-DHAS_ROCTRACER") + target_compile_options(kineto_base PRIVATE "-D__HIP_PLATFORM_HCC__") + target_compile_options(kineto_base PRIVATE "-D__HIP_PLATFORM_AMD__") +endif() + +target_compile_options(kineto_base PRIVATE "${KINETO_COMPILE_OPTIONS}") +target_compile_options(kineto_api PRIVATE "${KINETO_COMPILE_OPTIONS}") + +if(NOT TARGET fmt) + if(NOT FMT_SOURCE_DIR) + set(FMT_SOURCE_DIR "${LIBKINETO_THIRDPARTY_DIR}/fmt" + CACHE STRING "fmt source directory from submodules") + endif() + + # Build FMT. + # FMT and some other libraries use BUILD_SHARED_LIBS to control + # the library type. + # Save and restore the value after configuring FMT + set(TEMP_BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS}) + set(BUILD_SHARED_LIBS OFF CACHE BOOL "Build shared libs" FORCE) + set(FMT_LIBRARY_TYPE static CACHE STRING "Set lib type to static") + add_subdirectory("${FMT_SOURCE_DIR}" "${LIBKINETO_BINARY_DIR}/fmt") + set_property(TARGET fmt PROPERTY POSITION_INDEPENDENT_CODE ON) + set(BUILD_SHARED_LIBS ${TEMP_BUILD_SHARED_LIBS} CACHE BOOL "Build shared libs" FORCE) +endif() + +set(FMT_INCLUDE_DIR "${FMT_SOURCE_DIR}/include") +message(STATUS "Kineto: FMT_SOURCE_DIR = ${FMT_SOURCE_DIR}") +message(STATUS "Kineto: FMT_INCLUDE_DIR = ${FMT_INCLUDE_DIR}") +if (NOT CUPTI_INCLUDE_DIR) + set(CUPTI_INCLUDE_DIR "${CUDA_SOURCE_DIR}/extras/CUPTI/include") +endif() +if (NOT CUDA_INCLUDE_DIRS) + set(CUDA_INCLUDE_DIRS "${CUDA_SOURCE_DIR}/include") +endif() +if (NOT ROCTRACER_INCLUDE_DIR) + set(ROCTRACER_INCLUDE_DIR "${ROCM_SOURCE_DIR}/roctracer/include") +endif() +if (NOT ROCM_INCLUDE_DIRS) + set(ROCM_INCLUDE_DIRS "${ROCM_SOURCE_DIR}/include") +endif() + +message(INFO " CUPTI_INCLUDE_DIR = ${CUPTI_INCLUDE_DIR}") +message(INFO " ROCTRACER_INCLUDE_DIR = ${ROCTRACER_INCLUDE_DIR}") + +target_include_directories(kineto_base PUBLIC + $ + $ + $ + $ + $ + $ + $) + +target_include_directories(kineto_api PUBLIC + $ + $) + +if(KINETO_LIBRARY_TYPE STREQUAL "default") + add_library(kineto + $ + $) +elseif(KINETO_LIBRARY_TYPE STREQUAL "static") + add_library(kineto STATIC + $ + $) +elseif(KINETO_LIBRARY_TYPE STREQUAL "shared") + add_library(kineto SHARED + $) + set_property(TARGET kineto_base PROPERTY POSITION_INDEPENDENT_CODE ON) + set_target_properties(kineto PROPERTIES + CXX_VISIBILITY_PRESET hidden) +else() + message(FATAL_ERROR "Unsupported library type ${KINETO_LIBRARY_TYPE}") +endif() + +if(NOT LIBKINETO_NOROCTRACER) + find_library(ROCTRACER_LIBRARY NAMES libroctracer64.so HINTS /opt/rocm/roctracer/lib) + target_link_libraries(kineto "${ROCTRACER_LIBRARY}") + find_library(KINETO_HIP_LIBRARY NAMES libamdhip64.so HINTS /opt/rocm/lib) + target_link_libraries(kineto "${KINETO_HIP_LIBRARY}") +endif() + +if(NOT LIBKINETO_NOCUPTI) + target_link_libraries(kineto "${CUDA_cupti_LIBRARY}") +endif() +target_link_libraries(kineto $) +add_dependencies(kineto fmt::fmt-header-only) + +install(TARGETS kineto EXPORT kinetoLibraryConfig + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) + +install(FILES ${LIBKINETO_PUBLIC_HEADERS} + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/kineto") + +install(EXPORT kinetoLibraryConfig DESTINATION share/cmake/kineto + FILE kinetoLibraryConfig.cmake) + +if(KINETO_BUILD_TESTS) + add_subdirectory(test) +endif() diff --git a/tb_plugins/profiling/libkineto/README.md b/tb_plugins/profiling/libkineto/README.md new file mode 100644 index 000000000..37127ca5a --- /dev/null +++ b/tb_plugins/profiling/libkineto/README.md @@ -0,0 +1,65 @@ +# Libkineto + +Libkineto is an in-process profiling library, part of the Kineto performance +tools project. + +The library provides a way to collect GPU traces and metrics from the host +process, either via the library public API or by sending a signal, if enabled. + +Currently only NVIDIA GPUs are supported. + +## Build Notes +Libkineto uses the standard CMAKE-based build flow. + +### Dependencies +Libkineto requires gcc 5+ and: + +- NVIDIA CUPTI: used to collect traces and metrics from NVIDIA GPUs. +- fmt: used for its convenient and lightweight string formatting functionality. +- googletest: required to build and run Kineto's tests. + - **googletest is not required** if you don't want to run Kineto tests. +By default, building of tests is **on**. Turn it off by setting `KINETO_BUILD_TESTS` to **off**. + +You can download [NVIDIA CUPTI][1], [fmt][2], [googletest][3] and set +`CUDA_SOURCE_DIR`, `FMT_SOURCE_DIR`, `GOOGLETEST_SOURCE_DIR` respectively for +cmake to find these libraries. If the fmt and googletest variables are not set, cmake will +build the git submodules found in the `third_party` directory. +If `CUDA_SOURCE_DIR` is not set, libkineto will fail to build. + +### Building Libkineto + +``` +# Check out repo and sub modules +git clone --recursive https://github.com/pytorch/kineto.git +# Build libkineto with cmake +cd kineto/libkineto +mkdir build && cd build +cmake .. +make +``` + +To run the tests after building libkineto (if tests are built), use the following +command: +``` +make test +``` + +### Installing Libkineto +``` +make install +``` + +## How Libkineto works +We will provide a high-level overview, design philosophy and brief descriptions of various +parts of Libkineto in upcoming blogs. + +## Full documentation +We strive to keep our source files readable. The best and up-to-date +documentation is available in the source files. + +## License +Libkineto is BSD licensed, as detailed in the [LICENSE](../LICENSE) file. + +[1]:https://developer.nvidia.com/CUPTI-CTK10_2 +[2]:https://github.com/fmt +[3]:https://github.com/google/googletest diff --git a/tb_plugins/profiling/libkineto/include/AbstractConfig.h b/tb_plugins/profiling/libkineto/include/AbstractConfig.h new file mode 100644 index 000000000..1cadf4906 --- /dev/null +++ b/tb_plugins/profiling/libkineto/include/AbstractConfig.h @@ -0,0 +1,113 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#pragma once + +#include +#include +#include +#include +#include + +namespace KINETO_NAMESPACE { + +class AbstractConfig { + public: + AbstractConfig& operator=(const AbstractConfig&) = delete; + AbstractConfig(AbstractConfig&&) = delete; + AbstractConfig& operator=(AbstractConfig&&) = delete; + + virtual ~AbstractConfig() { + for (const auto& p : featureConfigs_) { + delete p.second; + } + } + + // Return a copy of the full derived class + virtual AbstractConfig* cloneDerived(AbstractConfig& parent) const = 0; + + // Returns true if successfully parsed the config string + bool parse(const std::string& conf); + + // Default setup for signal-triggered profiling + virtual void setSignalDefaults() { + for (auto& p : featureConfigs_) { + p.second->setSignalDefaults(); + } + } + + // Default setup for client-triggered profiling + virtual void setClientDefaults() { + for (auto& p : featureConfigs_) { + p.second->setClientDefaults(); + } + } + + // Time config was created / updated + std::chrono::time_point timestamp() const { + return timestamp_; + } + + // Source config string that this was parsed from + const std::string& source() const { + return source_; + } + + AbstractConfig& feature(std::string name) const { + const auto& pos = featureConfigs_.find(name); + return *pos->second; + } + + // Transfers ownership of cfg arg + void addFeature(const std::string& name, AbstractConfig* cfg) { + featureConfigs_[name] = cfg; + } + + protected: + AbstractConfig() {} + AbstractConfig(const AbstractConfig& other) = default; + + // Return true if the option was recognized and successfully parsed. + // Throw std::invalid_argument if val is invalid. + virtual bool handleOption(const std::string& name, std::string& val); + + // Perform post-validation checks, typically conditons involving + // multiple options. + // Throw std::invalid_argument if automatic correction can not be made. + // + // @param fallbackProfileStartTime Specify a fallback profile start timestamp in case it was never specified by the client + virtual void validate(const std::chrono::time_point& fallbackProfileStartTime) = 0; + + // TODO: Separate out each profiler type into features? + virtual void printActivityProfilerConfig(std::ostream& s) const; + + // Helpers for use in handleOption + // Split a string by delimiter and remove external white space + std::vector splitAndTrim(const std::string& s, char delim) const; + // Lowercase for case-insensitive comparisons + std::string toLower(std::string& s) const; + // Does string end with suffix + bool endsWith(const std::string& s, const std::string& suffix) const; + // Conversions + int64_t toIntRange(const std::string& val, int64_t min, int64_t max) const; + int32_t toInt32(const std::string& val) const; + int64_t toInt64(const std::string& val) const; + bool toBool(std::string& val) const; + + void cloneFeaturesInto(AbstractConfig& cfg) const { + for (const auto& feature : featureConfigs_) { + cfg.featureConfigs_[feature.first] = feature.second->cloneDerived(cfg); + } + } + + private: + // Time config was created / updated + std::chrono::time_point timestamp_{}; + + // Original configuration string, used for comparison + std::string source_{""}; + + // Configuration objects for optional features + std::map featureConfigs_{}; +}; + +} // namespace KINETO_NAMESPACE diff --git a/tb_plugins/profiling/libkineto/include/ActivityProfilerInterface.h b/tb_plugins/profiling/libkineto/include/ActivityProfilerInterface.h new file mode 100644 index 000000000..29871e47a --- /dev/null +++ b/tb_plugins/profiling/libkineto/include/ActivityProfilerInterface.h @@ -0,0 +1,91 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#pragma once + +#include +#include +#include +#include + +#include "ActivityType.h" +#include "ActivityTraceInterface.h" +#include "IActivityProfiler.h" + +namespace libkineto { + +class ActivityProfilerController; +struct CpuTraceBuffer; +class Config; + +class ActivityProfilerInterface { + + public: + virtual ~ActivityProfilerInterface() {}; + + virtual void init() {} + virtual bool isInitialized() { + return false; + } + virtual bool isActive(){ + return false; + } + + // *** Asynchronous API *** + // Instead of starting and stopping the trace manually, provide a start time + // and duration and / or iteration stop criterion. + // Tracing terminates when either condition is met. + virtual void scheduleTrace(const std::string& configStr) {} + + // *** Synchronous API *** + // These must be called in order: + // prepareTrace -> startTrace -> stopTrace. + + // Many tracing structures are lazily initialized during trace collection, + // with potentially high overhead. + // Call prepareTrace to enable tracing, then run the region to trace + // at least once (and ideally run the same code that is to be traced) to + // allow tracing structures to be initialized. + virtual void prepareTrace( + const std::set& activityTypes, + const std::string& configStr = "") {} + + // Start recording, potentially reusing any buffers allocated since + // prepareTrace was called. + virtual void startTrace() {} + + // Stop and process trace, producing an in-memory list of trace records. + // The processing will be done synchronously (using the calling thread.) + virtual std::unique_ptr stopTrace() { + return nullptr; + } + + // Re-evaluate internal state to allow for triggering operations based + // on number of iteration. each implicitly increments the iteration count + virtual void step() {} + + // *** TraceActivity API *** + // FIXME: Pass activityProfiler interface into clientInterface? + virtual void pushCorrelationId(uint64_t id){} + virtual void popCorrelationId(){} + virtual void transferCpuTrace( + std::unique_ptr traceBuffer){} + + // Correlation ids for user defined spans + virtual void pushUserCorrelationId(uint64_t){} + virtual void popUserCorrelationId(){} + + // Saves information for the current thread to be used in profiler output + // Client must record any new kernel thread where the activity has occured. + virtual void recordThreadInfo() {} + + // Record trace metadata, currently supporting only string key and values, + // values with the same key are overwritten + virtual void addMetadata(const std::string& key, const std::string& value) = 0; + + // Add a child activity profiler, this enables frameworks in the application + // to enable custom framework events. + virtual void addChildActivityProfiler( + std::unique_ptr profiler) {} +}; + +} // namespace libkineto diff --git a/tb_plugins/profiling/libkineto/include/ActivityTraceInterface.h b/tb_plugins/profiling/libkineto/include/ActivityTraceInterface.h new file mode 100644 index 000000000..23d4edab0 --- /dev/null +++ b/tb_plugins/profiling/libkineto/include/ActivityTraceInterface.h @@ -0,0 +1,21 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#pragma once + +#include +#include + +namespace libkineto { + +struct ITraceActivity; + +class ActivityTraceInterface { + public: + virtual ~ActivityTraceInterface() {} + virtual const std::vector* activities() { + return nullptr; + } + virtual void save(const std::string& path) {} +}; + +} // namespace libkineto diff --git a/tb_plugins/profiling/libkineto/include/ActivityType.h b/tb_plugins/profiling/libkineto/include/ActivityType.h new file mode 100644 index 000000000..74c6a2531 --- /dev/null +++ b/tb_plugins/profiling/libkineto/include/ActivityType.h @@ -0,0 +1,34 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#pragma once + +#include +#include + +namespace libkineto { + +enum class ActivityType { + CPU_OP = 0, // cpu side ops + USER_ANNOTATION, + GPU_USER_ANNOTATION, + GPU_MEMCPY, + GPU_MEMSET, + CONCURRENT_KERNEL, // on-device kernels + EXTERNAL_CORRELATION, + CUDA_RUNTIME, // host side cuda runtime events + CUDA_PROFILER_RANGE, // CUPTI Profiler range for performance metrics + GLOW_RUNTIME, // host side glow runtime events + CPU_INSTANT_EVENT, // host side point-like events + PYTHON_FUNCTION, + OVERHEAD, // CUPTI induced overhead events sampled from its overhead API. + ENUM_COUNT // This is to add buffer and not used for any profiling logic. Add your new type before it. +}; + +const char* toString(ActivityType t); +ActivityType toActivityType(const std::string& str); + +// Return an array of all activity types except COUNT +constexpr int activityTypeCount = (int)ActivityType::ENUM_COUNT; +const std::array activityTypes(); + +} // namespace libkineto diff --git a/tb_plugins/profiling/libkineto/include/ClientInterface.h b/tb_plugins/profiling/libkineto/include/ClientInterface.h new file mode 100644 index 000000000..06dc07583 --- /dev/null +++ b/tb_plugins/profiling/libkineto/include/ClientInterface.h @@ -0,0 +1,16 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#pragma once + +namespace libkineto { + +class ClientInterface { + public: + virtual ~ClientInterface() {} + virtual void init() = 0; + virtual void warmup(bool setupOpInputsCollection) = 0; + virtual void start() = 0; + virtual void stop() = 0; +}; + +} // namespace libkineto diff --git a/tb_plugins/profiling/libkineto/include/Config.h b/tb_plugins/profiling/libkineto/include/Config.h new file mode 100644 index 000000000..040e96c9f --- /dev/null +++ b/tb_plugins/profiling/libkineto/include/Config.h @@ -0,0 +1,433 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#pragma once + +#include "AbstractConfig.h" +#include "ActivityType.h" + +#include +#include +#include +#include +#include +#include + +namespace KINETO_NAMESPACE { + +using namespace libkineto; + +class Config : public AbstractConfig { + public: + Config(); + Config& operator=(const Config&) = delete; + Config(Config&&) = delete; + Config& operator=(Config&&) = delete; + + // Return a full copy including feature config object + std::unique_ptr clone() const { + auto cfg = std::unique_ptr(new Config(*this)); + cloneFeaturesInto(*cfg); + return cfg; + } + + bool handleOption(const std::string& name, std::string& val) override; + + void setClientDefaults() override; + + // Log events to this file + const std::string& eventLogFile() const { + return eventLogFile_; + } + + bool activityProfilerEnabled() const { + return activityProfilerEnabled_ || + activitiesOnDemandTimestamp_.time_since_epoch().count() > 0; + } + + // Log activitiy trace to this file + const std::string& activitiesLogFile() const { + return activitiesLogFile_; + } + + // Log activitiy trace to this url + const std::string& activitiesLogUrl() const { + return activitiesLogUrl_; + } + + void setActivitiesLogUrl(const std::string& url) { + activitiesLogUrl_ = url; + } + + bool activitiesLogToMemory() const { + return activitiesLogToMemory_; + } + + // Is profiling enabled for the given device? + bool eventProfilerEnabledForDevice(uint32_t dev) const { + return 0 != (eventProfilerDeviceMask_ & (1 << dev)); + } + + // Take a sample (read hardware counters) at this frequency. + // This controls how often counters are read - if all counters cannot + // be collected simultaneously then multiple samples are needed to + // collect all requested counters - see multiplex period. + std::chrono::milliseconds samplePeriod() const { + return samplePeriod_; + } + + void setSamplePeriod(std::chrono::milliseconds period) { + samplePeriod_ = period; + } + + // When all requested counters cannot be collected simultaneously, + // counters will be multiplexed at this frequency. + // Multiplexing can have a large performance impact if done frequently. + // To avoid a perf impact, keep this at 1s or above. + std::chrono::milliseconds multiplexPeriod() const { + return multiplexPeriod_; + } + + void setMultiplexPeriod(std::chrono::milliseconds period) { + multiplexPeriod_ = period; + } + + // Report counters at this frequency. Note that several samples can + // be reported each time, see samplesPerReport. + std::chrono::milliseconds reportPeriod() const { + return reportPeriod_; + } + + void setReportPeriod(std::chrono::milliseconds msecs); + + // Number of samples dispatched each report period. + // Must be in the range [1, report period / sample period]. + // In other words, aggregation is supported but not interpolation. + int samplesPerReport() const { + return samplesPerReport_; + } + + void setSamplesPerReport(int count) { + samplesPerReport_ = count; + } + + // The names of events to collect + const std::set& eventNames() const { + return eventNames_; + } + + // Add additional events to be profiled + void addEvents(const std::set& names) { + eventNames_.insert(names.begin(), names.end()); + } + + // The names of metrics to collect + const std::set& metricNames() const { + return metricNames_; + } + + // Add additional metrics to be profiled + void addMetrics(const std::set& names) { + metricNames_.insert(names.begin(), names.end()); + } + + const std::vector& percentiles() const { + return eventReportPercentiles_; + } + + // Profile for this long, then revert to base config + std::chrono::seconds eventProfilerOnDemandDuration() const { + return eventProfilerOnDemandDuration_; + } + + void setEventProfilerOnDemandDuration(std::chrono::seconds duration) { + eventProfilerOnDemandDuration_ = duration; + } + + // Too many event profilers on a single system can overload the driver. + // At some point, latencies shoot through the roof and collection of samples + // becomes impossible. To avoid this situation we have a limit of profilers + // per GPU. + // NOTE: Communication with a daemon is needed for this feature. + // Library must be built with an active DaemonConfigLoader. + int maxEventProfilersPerGpu() const { + return eventProfilerMaxInstancesPerGpu_; + } + + // On Cuda11 we've seen occasional hangs when reprogramming counters + // Monitor profiling threads and report when a thread is not responding + // for a given number of seconds. + // A period of 0 means disable. + std::chrono::seconds eventProfilerHeartbeatMonitorPeriod() const { + return eventProfilerHeartbeatMonitorPeriod_; + } + + // The types of activities selected in the configuration file + const std::set& selectedActivityTypes() const { + return selectedActivityTypes_; + } + + void setSelectedActivityTypes(const std::set& types) { + selectedActivityTypes_ = types; + } + + bool isOpInputsCollectionEnabled() const { + return enableOpInputsCollection_; + } + + // Trace for this long + std::chrono::milliseconds activitiesDuration() const { + return activitiesDuration_; + } + + // Trace for this many iterations, determined by external API + int activitiesRunIterations() const { + return activitiesRunIterations_; + } + + std::chrono::milliseconds activitiesDurationDefault() const; + + void setActivitiesDuration(std::chrono::milliseconds duration) { + activitiesDuration_ = duration; + } + + int activitiesMaxGpuBufferSize() const { + return activitiesMaxGpuBufferSize_; + } + + std::chrono::seconds activitiesWarmupDuration() const { + return activitiesWarmupDuration_; + } + + int activitiesWarmupIterations() const { + return activitiesWarmupIterations_; + } + + // Timestamp at which the profiling to start, requested by the user. + const std::chrono::time_point requestTimestamp() + const { + if (profileStartTime_.time_since_epoch().count()) { + return profileStartTime_; + } + + // TODO(T94634890): Deperecate requestTimestamp + return requestTimestamp_ + maxRequestAge() + activitiesWarmupDuration(); + } + + bool hasProfileStartTime() const { + return requestTimestamp_.time_since_epoch().count() > 0 || + profileStartTime_.time_since_epoch().count() > 0; + } + + int profileStartIteration() const { + return profileStartIteration_; + } + + bool hasProfileStartIteration() const { + return profileStartIteration_ >= 0 && activitiesRunIterations_ > 0; + } + + void setProfileStartIteration(int iter) { + profileStartIteration_ = iter; + } + + int profileStartIterationRoundUp() const { + return profileStartIterationRoundUp_; + } + + // calculate the start iteration accounting for warmup + int startIterationIncludingWarmup() const { + if (!hasProfileStartIteration()) { + return -1; + } + return profileStartIteration_ - activitiesWarmupIterations_; + } + + const std::chrono::seconds maxRequestAge() const; + + // All VLOG* macros will log if the verbose log level is >= + // the verbosity specified for the verbose log message. + // Default value is -1, so messages with log level 0 will log by default. + int verboseLogLevel() const { + return verboseLogLevel_; + } + + // Modules for which verbose logging is enabled. + // If empty, logging is enabled for all modules. + const std::vector& verboseLogModules() const { + return verboseLogModules_; + } + + bool sigUsr2Enabled() const { + return enableSigUsr2_; + } + + bool ipcFabricEnabled() const { + return enableIpcFabric_; + } + + static std::chrono::milliseconds alignUp( + std::chrono::milliseconds duration, + std::chrono::milliseconds alignment) { + duration += alignment; + return duration - (duration % alignment); + } + + std::chrono::time_point + eventProfilerOnDemandStartTime() const { + return eventProfilerOnDemandTimestamp_; + } + + std::chrono::time_point + eventProfilerOnDemandEndTime() const { + return eventProfilerOnDemandTimestamp_ + eventProfilerOnDemandDuration_; + } + + std::chrono::time_point + activityProfilerRequestReceivedTime() const { + return activitiesOnDemandTimestamp_; + } + + // Users may request and set trace id and group trace id. + const std::string& requestTraceID() const { + return requestTraceID_; + } + + void setRequestTraceID(const std::string& tid) { + requestTraceID_ = tid; + } + + const std::string& requestGroupTraceID() const { + return requestGroupTraceID_; + } + + void setRequestGroupTraceID(const std::string& gtid) { + requestGroupTraceID_ = gtid; + } + + void updateActivityProfilerRequestReceivedTime(); + + void printActivityProfilerConfig(std::ostream& s) const override; + + void validate( + const std::chrono::time_point& fallbackProfileStartTime) override; + + static void addConfigFactory( + std::string name, + std::function factory); + + void print(std::ostream& s) const; + + private: + explicit Config(const Config& other) = default; + + AbstractConfig* cloneDerived(AbstractConfig& parent) const override { + // Clone from AbstractConfig not supported + assert(false); + return nullptr; + } + + uint8_t createDeviceMask(const std::string& val); + + // Adds valid activity types from the user defined string list in the + // configuration file + void setActivityTypes(const std::vector& selected_activities); + + // Sets the default activity types to be traced + void selectDefaultActivityTypes() { + // If the user has not specified an activity list, add all types + for (ActivityType t : activityTypes()) { + // Do no enable this by default + // TODO: introduce optional types + if (t != ActivityType::OVERHEAD) { + selectedActivityTypes_.insert(t); + } + } + } + + int verboseLogLevel_; + std::vector verboseLogModules_; + + // Event profiler + // These settings are also supported in on-demand mode + std::chrono::milliseconds samplePeriod_; + std::chrono::milliseconds reportPeriod_; + int samplesPerReport_; + std::set eventNames_; + std::set metricNames_; + + // On-demand duration + std::chrono::seconds eventProfilerOnDemandDuration_; + // Last on-demand request + std::chrono::time_point + eventProfilerOnDemandTimestamp_; + + int eventProfilerMaxInstancesPerGpu_; + + // Monitor whether event profiler threads are stuck + // at this frequency + std::chrono::seconds eventProfilerHeartbeatMonitorPeriod_; + + // These settings can not be changed on-demand + std::string eventLogFile_; + std::vector eventReportPercentiles_ = {5, 25, 50, 75, 95}; + uint8_t eventProfilerDeviceMask_ = ~0; + std::chrono::milliseconds multiplexPeriod_; + + // Activity profiler + bool activityProfilerEnabled_; + std::set selectedActivityTypes_; + + // The activity profiler settings are all on-demand + std::string activitiesLogFile_; + + std::string activitiesLogUrl_; + + // Log activities to memory buffer + bool activitiesLogToMemory_{false}; + + int activitiesMaxGpuBufferSize_; + std::chrono::seconds activitiesWarmupDuration_; + int activitiesWarmupIterations_; + + // Client Interface + // Enable inputs collection when tracing ops + bool enableOpInputsCollection_{true}; + + // Profile for specified iterations and duration + std::chrono::milliseconds activitiesDuration_; + int activitiesRunIterations_; + + // Below are not used + // Use this net name for iteration count + std::string activitiesExternalAPIIterationsTarget_; + // Only profile nets that includes this in the name + std::vector activitiesExternalAPIFilter_; + // Only profile nets with at least this many operators + int activitiesExternalAPINetSizeThreshold_; + // Only profile nets with at least this many GPU operators + int activitiesExternalAPIGpuOpCountThreshold_; + // Last activity profiler request + std::chrono::time_point + activitiesOnDemandTimestamp_; + + // Synchronized start timestamp + std::chrono::time_point profileStartTime_; + // or start iteration + int profileStartIteration_; + int profileStartIterationRoundUp_; + + // DEPRECATED + std::chrono::time_point requestTimestamp_; + + // Enable profiling via SIGUSR2 + bool enableSigUsr2_; + + // Enable IPC Fabric instead of thrift communication + bool enableIpcFabric_; + + // Logger Metadata + std::string requestTraceID_; + std::string requestGroupTraceID_; +}; + +} // namespace KINETO_NAMESPACE diff --git a/tb_plugins/profiling/libkineto/include/GenericTraceActivity.h b/tb_plugins/profiling/libkineto/include/GenericTraceActivity.h new file mode 100644 index 000000000..4272cf1ef --- /dev/null +++ b/tb_plugins/profiling/libkineto/include/GenericTraceActivity.h @@ -0,0 +1,125 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#pragma once + +#include +#include +#include +#include + +#include "ThreadUtil.h" +#include "ITraceActivity.h" +#include "TraceSpan.h" + +namespace libkineto { + +// Link type, used in GenericTraceActivity.flow.type +constexpr unsigned int kLinkFwdBwd = 1; +constexpr unsigned int kLinkAsyncCpuGpu = 2; + +// @lint-ignore-every CLANGTIDY cppcoreguidelines-non-private-member-variables-in-classes +// @lint-ignore-every CLANGTIDY cppcoreguidelines-pro-type-member-init +class GenericTraceActivity : public ITraceActivity { + + public: + GenericTraceActivity() : activityType(ActivityType::ENUM_COUNT), traceSpan_(NULL) {} + + GenericTraceActivity( + const TraceSpan& trace, ActivityType type, const std::string& name) + : activityType(type), activityName(name), traceSpan_(&trace) { + } + + int64_t deviceId() const override { + return device; + } + + int64_t resourceId() const override { + return resource; + } + + int32_t getThreadId() const override { + return threadId; + } + + int64_t timestamp() const override { + return startTime; + } + + int64_t duration() const override { + return endTime - startTime; + } + + int64_t correlationId() const override { + return id; + } + + ActivityType type() const override { + return activityType; + } + + const ITraceActivity* linkedActivity() const override { + return nullptr; + } + + int flowType() const override { + return flow.type; + } + + int flowId() const override { + return flow.id; + } + + bool flowStart() const override { + return flow.start; + } + + const std::string name() const override { + return activityName; + } + + const TraceSpan* traceSpan() const override { + return traceSpan_; + } + + void log(ActivityLogger& logger) const override; + + //Encode client side metadata as a key/value + template + void addMetadata(const std::string& key, const ValType& value) { + metadata_.push_back(fmt::format("\"{}\": {}", key, value)); + } + + void addMetadataQuoted(const std::string& key, const std::string& value) { + metadata_.push_back(fmt::format("\"{}\": \"{}\"", key, value)); + } + + const std::string metadataJson() const override { + return fmt::format("{}", fmt::join(metadata_, ", ")); + } + + virtual ~GenericTraceActivity() {}; + + int64_t startTime{0}; + int64_t endTime{0}; + int32_t id{0}; + int32_t device{0}; + int32_t resource{0}; + int32_t threadId{0}; + ActivityType activityType; + std::string activityName; + struct Flow { + Flow(): id(0), type(0), start(0) {} + // Ids must be unique within each type + uint32_t id : 27; + // Type will be used to connect flows between profilers, as + // well as look up flow information (name etc) + uint32_t type : 4; + uint32_t start : 1; + } flow; + + private: + const TraceSpan* traceSpan_; + std::vector metadata_; +}; + +} // namespace libkineto diff --git a/tb_plugins/profiling/libkineto/include/IActivityProfiler.h b/tb_plugins/profiling/libkineto/include/IActivityProfiler.h new file mode 100644 index 000000000..f5d4b3fb8 --- /dev/null +++ b/tb_plugins/profiling/libkineto/include/IActivityProfiler.h @@ -0,0 +1,104 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#pragma once + +#include +#include +#include + +#include "Config.h" +#include "GenericTraceActivity.h" + +/* This file includes an abstract base class for an activity profiler + * that can be implemented by multiple tracing agents in the application. + * The high level Kineto profiler can co-ordinate start and end of tracing + * and combine together events from multiple such activity profilers. + */ + +namespace libkineto { + +using namespace KINETO_NAMESPACE; + +#ifdef _MSC_VER +// workaround for the predefined ERROR macro on Windows +#undef ERROR +#endif // _MSC_VER + +enum class TraceStatus { + READY, // Accepting trace requests + WARMUP, // Performing trace warmup + RECORDING, // Actively collecting activities + PROCESSING, // Recording is complete, preparing results + ERROR, // One or more errors (and possibly also warnings) occurred. + WARNING, // One or more warnings occurred. +}; + +/* IActivityProfilerSession: + * an opaque object that can be used by a high level profiler to + * start/stop and return trace events. + */ +class IActivityProfilerSession { + + public: + virtual ~IActivityProfilerSession() {} + + // start the trace collection synchronously + virtual void start() = 0; + + // stop the trace collection synchronously + virtual void stop() = 0; + + TraceStatus status() { + return status_; + } + + // returns list of Trace Activities + virtual std::vector& activities() = 0; + + // returns errors with this trace + virtual std::vector errors() = 0; + + // processes trace activities using logger + virtual void processTrace(ActivityLogger& logger) = 0; + + // XXX define trace formats + // virtual save(string name, TraceFormat format) + + protected: + TraceStatus status_ = TraceStatus::READY; +}; + + +/* Activity Profiler Plugins: + * These allow other frameworks to integrate into Kineto's primariy + * activity profiler. While the primary activity profiler handles + * timing the trace collections and correlating events the plugins + * can become source of new trace activity types. + */ +class IActivityProfiler { + + public: + + virtual ~IActivityProfiler() {} + + // name of profiler + virtual const std::string& name() const = 0; + + // returns activity types this profiler supports + virtual const std::set& availableActivities() const = 0; + + // Calls prepare() on registered tracer providers passing in the relevant + // activity types. Returns a profiler session handle + virtual std::unique_ptr configure( + const std::set& activity_types, + const Config& config) = 0; + + // asynchronous version of the above with future timestamp and duration. + virtual std::unique_ptr configure( + int64_t ts_ms, + int64_t duration_ms, + const std::set& activity_types, + const Config& config) = 0; +}; + +} // namespace libkineto diff --git a/tb_plugins/profiling/libkineto/include/ILoggerObserver.h b/tb_plugins/profiling/libkineto/include/ILoggerObserver.h new file mode 100644 index 000000000..4fce7851b --- /dev/null +++ b/tb_plugins/profiling/libkineto/include/ILoggerObserver.h @@ -0,0 +1,50 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#pragma once + +#include + +// Stages in libkineto used when pushing logs to UST Logger. +constexpr char kWarmUpStage[] = "Warm Up"; +constexpr char kCollectionStage[] = "Collection"; +constexpr char kPostProcessingStage[] = "Post Processing"; + +#if !USE_GOOGLE_LOG + +#include +#include + +namespace libkineto { + +enum LoggerOutputType { + VERBOSE = 0, + INFO = 1, + WARNING = 2, + ERROR = 3, + STAGE = 4, + ENUM_COUNT = 5 +}; + +const char* toString(LoggerOutputType t); +LoggerOutputType toLoggerOutputType(const std::string& str); + +constexpr int LoggerTypeCount = (int) LoggerOutputType::ENUM_COUNT; + +class ILoggerObserver { + public: + virtual ~ILoggerObserver() = default; + virtual void write(const std::string& message, LoggerOutputType ot) = 0; + virtual const std::map> extractCollectorMetadata() = 0; + virtual void reset() = 0; + virtual void addDevice(const int64_t device) = 0; + virtual void setTraceDurationMS(const int64_t duration) = 0; + virtual void addEventCount(const int64_t count) = 0; + virtual void setTraceID(const std::string&) {} + virtual void setGroupTraceID(const std::string&) {} + virtual void addDestination(const std::string& dest) = 0; + +}; + +} // namespace libkineto + +#endif // !USE_GOOGLE_LOG diff --git a/tb_plugins/profiling/libkineto/include/ITraceActivity.h b/tb_plugins/profiling/libkineto/include/ITraceActivity.h new file mode 100644 index 000000000..a477ed814 --- /dev/null +++ b/tb_plugins/profiling/libkineto/include/ITraceActivity.h @@ -0,0 +1,53 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#pragma once + +#include + +#include "ActivityType.h" + +namespace libkineto { + +class ActivityLogger; +struct TraceSpan; + +// Generic activity interface is borrowed from tensorboard protobuf format. +struct ITraceActivity { + virtual ~ITraceActivity() {} + // Device is a physical or logical entity, e.g. CPU, GPU or process + virtual int64_t deviceId() const = 0; + // A resource is something on the device, h/w thread, + // functional units etc. + virtual int64_t resourceId() const = 0; + // s/w thread + virtual int32_t getThreadId() const = 0; + // Start timestamp in mucrosecond + virtual int64_t timestamp() const = 0; + // Duration in microseconds + virtual int64_t duration() const = 0; + // Used to link up async activities + virtual int64_t correlationId() const = 0; + // Part of a flow, identified by flow id and type + virtual int flowType() const = 0; + virtual int flowId() const = 0; + virtual bool flowStart() const = 0; + virtual ActivityType type() const = 0; + virtual const std::string name() const = 0; + // Optional linked activity + virtual const ITraceActivity* linkedActivity() const = 0; + // Optional containing trace object + virtual const TraceSpan* traceSpan() const = 0; + // Log activity + virtual void log(ActivityLogger& logger) const = 0; + // Return json formatted metadata + // FIXME: Return iterator to dynamic type map here instead + virtual const std::string metadataJson() const = 0; + + static int64_t nsToUs(int64_t ns) { + // It's important that this conversion is the same everywhere. + // No rounding! + return ns / 1000; + } +}; + +} // namespace libkineto diff --git a/tb_plugins/profiling/libkineto/include/ThreadUtil.h b/tb_plugins/profiling/libkineto/include/ThreadUtil.h new file mode 100644 index 000000000..d1dc80ad2 --- /dev/null +++ b/tb_plugins/profiling/libkineto/include/ThreadUtil.h @@ -0,0 +1,22 @@ +#pragma once + +#include +#include +#include +#include + +namespace libkineto { + +int32_t systemThreadId(); +int32_t threadId(); +bool setThreadName(const std::string& name); +std::string getThreadName(); + +int32_t processId(); +std::string processName(int32_t pid); + +// Return a list of pids and process names for the current process +// and its parents. +std::vector> pidCommandPairsOfAncestors(); + +} // namespace libkineto diff --git a/tb_plugins/profiling/libkineto/include/TraceSpan.h b/tb_plugins/profiling/libkineto/include/TraceSpan.h new file mode 100644 index 000000000..af9a9d5ee --- /dev/null +++ b/tb_plugins/profiling/libkineto/include/TraceSpan.h @@ -0,0 +1,36 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#pragma once + +#include +#include +#include + +namespace libkineto { + +struct TraceSpan { + TraceSpan() = delete; + TraceSpan( + int64_t startTime, int64_t endTime, std::string name) + : startTime(startTime), endTime(endTime), name(std::move(name)) { + } + TraceSpan( + int opCount, int it, std::string name, std::string prefix) + : opCount(opCount), + iteration(it), + name(std::move(name)), + prefix(std::move(prefix)) { + } + + // FIXME: change to duration? + int64_t startTime{0}; + int64_t endTime{0}; + int opCount{0}; + int iteration{-1}; + // Name is used to identify timeline + std::string name; + // Prefix used to distinguish trace spans on the same timeline + std::string prefix; +}; + +} // namespace libkineto diff --git a/tb_plugins/profiling/libkineto/include/libkineto.h b/tb_plugins/profiling/libkineto/include/libkineto.h new file mode 100644 index 000000000..87c3d64f6 --- /dev/null +++ b/tb_plugins/profiling/libkineto/include/libkineto.h @@ -0,0 +1,138 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +// Mediator for initialization and profiler control + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ActivityProfilerInterface.h" +#include "ActivityType.h" +#include "ClientInterface.h" +#include "GenericTraceActivity.h" +#include "TraceSpan.h" +#include "IActivityProfiler.h" +#include "ActivityTraceInterface.h" + +#include "ThreadUtil.h" + +extern "C" { + void suppressLibkinetoLogMessages(); + int InitializeInjection(void); + bool libkineto_init(bool cpuOnly, bool logOnError); +} + +namespace libkineto { + +class Config; +class ConfigLoader; + +struct CpuTraceBuffer { + TraceSpan span{0, 0, "none"}; + int gpuOpCount; + std::deque activities; +}; + +using ChildActivityProfilerFactory = + std::function()>; + +class LibkinetoApi { + public: + + explicit LibkinetoApi(ConfigLoader& configLoader) + : configLoader_(configLoader) { + } + + // Called by client that supports tracing API. + // libkineto can still function without this. + void registerClient(ClientInterface* client); + + // Called by libkineto on init + void registerProfiler(std::unique_ptr profiler) { + activityProfiler_ = std::move(profiler); + initClientIfRegistered(); + } + + ActivityProfilerInterface& activityProfiler() { + return *activityProfiler_; + } + + ClientInterface* client() { + return client_; + } + + void initProfilerIfRegistered() { + static std::once_flag once; + if (activityProfiler_) { + std::call_once(once, [this] { + if (!activityProfiler_->isInitialized()) { + activityProfiler_->init(); + initChildActivityProfilers(); + } + }); + } + } + + bool isProfilerInitialized() const { + return activityProfiler_ && activityProfiler_->isInitialized(); + } + + bool isProfilerRegistered() const { + return activityProfiler_ != nullptr; + } + + void suppressLogMessages() { + suppressLibkinetoLogMessages(); + } + + // Provides access to profier configuration manaegement + ConfigLoader& configLoader() { + return configLoader_; + } + + void registerProfilerFactory( + ChildActivityProfilerFactory factory) { + if (isProfilerInitialized()) { + activityProfiler_->addChildActivityProfiler(factory()); + } else { + childProfilerFactories_.push_back(factory); + } + } + + private: + + void initChildActivityProfilers() { + if (!isProfilerInitialized()) { + return; + } + for (const auto& factory : childProfilerFactories_) { + activityProfiler_->addChildActivityProfiler(factory()); + } + childProfilerFactories_.clear(); + } + + // Client is initialized once both it and libkineto has registered + void initClientIfRegistered(); + + ConfigLoader& configLoader_; + std::unique_ptr activityProfiler_{}; + ClientInterface* client_{}; + int32_t clientRegisterThread_{0}; + + bool isLoaded_{false}; + std::vector childProfilerFactories_; +}; + +// Singleton +LibkinetoApi& api(); + +} // namespace libkineto diff --git a/tb_plugins/profiling/libkineto/include/time_since_epoch.h b/tb_plugins/profiling/libkineto/include/time_since_epoch.h new file mode 100644 index 000000000..caa6b4d92 --- /dev/null +++ b/tb_plugins/profiling/libkineto/include/time_since_epoch.h @@ -0,0 +1,16 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#pragma once + +#include + +namespace libkineto { + +inline int64_t timeSinceEpoch( + const std::chrono::time_point& t) { + return std::chrono::duration_cast( + t.time_since_epoch()) + .count(); +} + +} // namespace libkineto diff --git a/tb_plugins/profiling/libkineto/libkineto_defs.bzl b/tb_plugins/profiling/libkineto/libkineto_defs.bzl new file mode 100644 index 000000000..330c54a22 --- /dev/null +++ b/tb_plugins/profiling/libkineto/libkineto_defs.bzl @@ -0,0 +1,77 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# All rights reserved. +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +def get_libkineto_api_srcs(): + return [ + "src/ThreadUtil.cpp", + "src/libkineto_api.cpp", + ] + +def get_libkineto_cupti_srcs(with_api = True): + return [ + "src/CudaDeviceProperties.cpp", + "src/CuptiActivityApi.cpp", + "src/CuptiActivityPlatform.cpp", + "src/CuptiCallbackApi.cpp", + "src/CuptiEventApi.cpp", + "src/CuptiMetricApi.cpp", + "src/CuptiRangeProfilerApi.cpp", + "src/Demangle.cpp", + "src/EventProfiler.cpp", + "src/EventProfilerController.cpp", + "src/WeakSymbols.cpp", + "src/cupti_strings.cpp", + ] + (get_libkineto_cpu_only_srcs(with_api)) + +def get_libkineto_roctracer_srcs(with_api = True): + return [ + "src/RoctracerActivityApi.cpp", + ] + (get_libkineto_cpu_only_srcs(with_api)) + +def get_libkineto_cpu_only_srcs(with_api = True): + return [ + "src/AbstractConfig.cpp", + "src/CuptiActivityProfiler.cpp", + "src/ActivityProfilerController.cpp", + "src/ActivityProfilerProxy.cpp", + "src/ActivityType.cpp", + "src/Config.cpp", + "src/ConfigLoader.cpp", + "src/CuptiActivityApi.cpp", + "src/Demangle.cpp", + "src/GenericTraceActivity.cpp", + "src/ILoggerObserver.cpp", + "src/Logger.cpp", + "src/init.cpp", + "src/output_csv.cpp", + "src/output_json.cpp", + ] + (get_libkineto_api_srcs() if with_api else []) + +def get_libkineto_public_headers(): + return [ + "include/AbstractConfig.h", + "include/ActivityProfilerInterface.h", + "include/ActivityType.h", + "include/Config.h", + "include/ClientInterface.h", + "include/GenericTraceActivity.h", + "include/GenericTraceActivity.h", + "include/IActivityProfiler.h", + "include/ILoggerObserver.h", + "include/ITraceActivity.h", + "include/TraceSpan.h", + "include/ThreadUtil.h", + "include/libkineto.h", + "include/time_since_epoch.h", + ] + +# kineto code should be updated to not have to +# suppress these warnings. +KINETO_COMPILER_FLAGS = [ + "-fexceptions", + "-Wno-deprecated-declarations", + "-Wno-unused-function", + "-Wno-unused-private-field", +] diff --git a/tb_plugins/profiling/libkineto/sample_programs/kineto_playground.cpp b/tb_plugins/profiling/libkineto/sample_programs/kineto_playground.cpp new file mode 100644 index 000000000..780047912 --- /dev/null +++ b/tb_plugins/profiling/libkineto/sample_programs/kineto_playground.cpp @@ -0,0 +1,38 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#include +#include +#include + +#include +#include + +#include "kineto/libkineto/sample_programs/kineto_playground.cuh" + +using namespace kineto; + +static const std::string kFileName = "/tmp/kineto_playground_trace.json"; + +int main() { + warmup(); + + // Kineto config + + // Empty types set defaults to all types + std::set types; + + auto& profiler = libkineto::api().activityProfiler(); + libkineto::api().initProfilerIfRegistered(); + profiler.prepareTrace(types); + + // Good to warm up after prepareTrace to get cupti initialization to settle + warmup(); + profiler.startTrace(); + playground(); + + auto trace = profiler.stopTrace(); + LOG(INFO) << "Stopped and processed trace. Got " << trace->activities()->size() << " activities."; + trace->save(kFileName); + return 0; +} + diff --git a/tb_plugins/profiling/libkineto/sample_programs/kineto_playground.cu b/tb_plugins/profiling/libkineto/sample_programs/kineto_playground.cu new file mode 100644 index 000000000..54c6f82ff --- /dev/null +++ b/tb_plugins/profiling/libkineto/sample_programs/kineto_playground.cu @@ -0,0 +1,60 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#include + +#include "kineto_playground.cuh" + + +namespace kineto { + +void warmup(void) { + // Inititalizing CUDA can take a while which we normally do not want to see in Kineto traces. + // This is done in various ways that take Kineto as dependency. This is our way of doing warmup + // for kineto_playground + size_t bytes = 1000; + float* mem = NULL; + auto error = cudaMalloc(&mem, bytes); + if (error != cudaSuccess) { + printf("cudaMalloc failed during kineto_playground warmup. error code: %d", error); + return; + } + + cudaFree(mem); +} + +void basicMemcpyMemset(void) { + size_t size = (1 << 8) * sizeof(float); + float *hostMemSrc, *deviceMem, *hostMemDst; + cudaError_t err; + + hostMemSrc = (float*)malloc(size); + hostMemDst = (float*)malloc(size); + err = cudaMalloc(&deviceMem, size); + if (err != cudaSuccess) { + printf("cudaMalloc failed during %s", __func__); + return; + } + + memset(hostMemSrc, 1, size); + cudaMemcpy(deviceMem, hostMemSrc, size, cudaMemcpyHostToDevice); + if (err != cudaSuccess) { + printf("cudaMemcpy failed during %s", __func__); + return; + } + + cudaMemcpy(hostMemDst, deviceMem, size, cudaMemcpyDeviceToHost); + if (err != cudaSuccess) { + printf("cudaMemcpy failed during %s", __func__); + return; + } + + free(hostMemSrc); + free(hostMemDst); + cudaFree(deviceMem); +} + +void playground(void) { + // Add your experimental CUDA implementation here. +} + +} diff --git a/tb_plugins/profiling/libkineto/sample_programs/kineto_playground.cuh b/tb_plugins/profiling/libkineto/sample_programs/kineto_playground.cuh new file mode 100644 index 000000000..54e1ee59a --- /dev/null +++ b/tb_plugins/profiling/libkineto/sample_programs/kineto_playground.cuh @@ -0,0 +1,18 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#pragma once + +#include + +namespace kineto { + +// Warms up CUDA before the tracing starts +void warmup(void); + +// Basic usage of cudaMemcpy and cudaMemset +void basicMemcpyMemset(void); + +// Your experimental code goes in here! +void playground(void); + +} diff --git a/tb_plugins/profiling/libkineto/src/AbstractConfig.cpp b/tb_plugins/profiling/libkineto/src/AbstractConfig.cpp new file mode 100644 index 000000000..d60ab43c9 --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/AbstractConfig.cpp @@ -0,0 +1,188 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#include "AbstractConfig.h" + +#include +#include +#include + +#include "Logger.h" + +using namespace std::chrono; + +using std::string; +using std::vector; + +namespace KINETO_NAMESPACE { + +constexpr char kWhitespace[] = "\t\n "; + +static bool isWhitespace(string& s) { + return s.find_first_not_of(kWhitespace) == string::npos; +} + +// Remove whitespace from both end of string +static inline string trim(string& s) { + if (s.empty()) { + return s; + } else if (isWhitespace(s)) { + return ""; + } + auto start = s.find_first_not_of(kWhitespace); + auto end = s.find_last_not_of(kWhitespace); + return s.substr(start, end - start + 1); +} + +// Helper function for split. +// Return the index of char d in string s. +// If not found, returns the length of the string. +static int find(const char* s, char delim) { + int i; + for (i = 0; s[i]; i++) { + if (s[i] == delim) { + break; + } + } + return i; +} + +// Split a string by delimiter +static vector split(const string& s, char delim) { + vector res; + const char* cs = s.c_str(); + for (int i = find(cs, delim); cs[i]; cs += i + 1, i = find(cs, delim)) { + res.emplace_back(cs, i); + } + res.emplace_back(cs); + return res; +} + +// Remove a trailing comment. +static inline string stripComment(const string& s) { + std::size_t pos = s.find("#"); + return s.substr(0, pos); +} + +string AbstractConfig::toLower(string& s) const { + string res = s; + for (int i = 0; i < res.size(); i++) { + if (res[i] >= 'A' && res[i] <= 'Z') { + res[i] += ('a' - 'A'); + } + } + return res; +} + +bool AbstractConfig::endsWith(const string& s, const string& suffix) const { + if (suffix.size() > s.size()) { + return false; + } + return s.compare(s.size() - suffix.size(), suffix.size(), suffix) == 0; +} + +vector AbstractConfig::splitAndTrim(const string& s, char delim) const { + auto res = split(s, delim); + for (string& x : res) { + x = trim(x); + } + return res; +} + +int64_t AbstractConfig::toIntRange(const string& val, int64_t min, int64_t max) + const { + char* invalid; + int64_t res = strtoll(val.c_str(), &invalid, 10); + if (val.empty() || *invalid) { + throw std::invalid_argument(fmt::format("Invalid integer: {}", val)); + } else if (res < min || res > max) { + throw std::invalid_argument(fmt::format( + "Invalid argument: {} - expected range [{}, {}]", res, min, max)); + } + return res; +} + +int32_t AbstractConfig::toInt32(const string& val) const { + return toIntRange(val, 0, ~0u / 2); +} + +int64_t AbstractConfig::toInt64(const string& val) const { + return toIntRange(val, 0, ~0ul / 2); +} + +bool AbstractConfig::toBool(string& val) const { + const std::array bool_vals{ + "n", "y", "no", "yes", "f", "t", "false", "true"}; + const string lower_val = toLower(val); + for (int i = 0; i < bool_vals.size(); i++) { + if (lower_val == bool_vals[i]) { + return i % 2; + } + } + throw std::invalid_argument(fmt::format("Invalid bool argument: {}", val)); + return false; +} + +bool AbstractConfig::parse(const string& conf) { + std::istringstream iss(conf); + string line; + + timestamp_ = system_clock::now(); + + // Read the string stream 1 line at a time to parse. + while (std::getline(iss, line)) { + line = stripComment(line); + if (isWhitespace(line)) { + continue; + } + vector key_val = splitAndTrim(line, '='); + if (key_val.size() != 2) { + LOG(ERROR) << "Invalid config line: " << line; + return false; + } else { + bool handled = false; + try { + handled = handleOption(key_val[0], key_val[1]); + if (!handled) { + for (auto& feature_cfg : featureConfigs_) { + if (feature_cfg.second->handleOption(key_val[0], key_val[1])) { + handled = true; + break; + } + } + } + } catch (const std::exception& e) { + LOG(ERROR) << "Failed to parse config line: " << line; + LOG(ERROR) << e.what(); + return false; + } + if (!handled) { + // This might be due to using a newer config option on an + // older binary where it is not supported. In this case, + // print a warning message - but it is expected to work! + LOG(WARNING) << "Unrecognized config line: " << line; + } + } + } + + validate(timestamp_); + + // Store original text, used to detect updates + source_ = conf; + timestamp_ = system_clock::now(); + return true; +} + +bool AbstractConfig::handleOption( + const std::string& /* unused */, + std::string& /* unused */) { + LOG(ERROR) << "handleOption unimplemented"; + return false; +} + +void AbstractConfig::printActivityProfilerConfig(std::ostream& s) const { + for (const auto& feature_cfg : featureConfigs_) { + feature_cfg.second->printActivityProfilerConfig(s); + } +} + +} // namespace KINETO_NAMESPACE diff --git a/tb_plugins/profiling/libkineto/src/ActivityBuffers.h b/tb_plugins/profiling/libkineto/src/ActivityBuffers.h new file mode 100644 index 000000000..157af8793 --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/ActivityBuffers.h @@ -0,0 +1,29 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#pragma once + + +#include +#include + +#include "libkineto.h" +#include "CuptiActivityBuffer.h" + +namespace KINETO_NAMESPACE { + +struct ActivityBuffers { + std::list> cpu; + std::unique_ptr gpu; + + // Add a wrapper object to the underlying struct stored in the buffer + template + const ITraceActivity& addActivityWrapper(const T& act) { + wrappers_.push_back(std::make_unique(act)); + return *wrappers_.back().get(); + } + + private: + std::vector> wrappers_; +}; + +} // namespace KINETO_NAMESPACE diff --git a/tb_plugins/profiling/libkineto/src/ActivityLoggerFactory.h b/tb_plugins/profiling/libkineto/src/ActivityLoggerFactory.h new file mode 100644 index 000000000..0d1bf642c --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/ActivityLoggerFactory.h @@ -0,0 +1,60 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace KINETO_NAMESPACE { + +class ActivityLogger; + +class ActivityLoggerFactory { + + public: + using FactoryFunc = + std::function(const std::string& url)>; + + // Add logger factory for a protocol prefix + void addProtocol(const std::string& protocol, FactoryFunc f) { + factories_[tolower(protocol)] = f; + } + + // Create a logger, invoking the factory for the protocol specified in url + std::unique_ptr makeLogger(const std::string& url) const { + std::string protocol = extractProtocol(url); + auto it = factories_.find(tolower(protocol)); + if (it != factories_.end()) { + return it->second(stripProtocol(url)); + } + throw std::invalid_argument(fmt::format( + "No logger registered for the {} protocol prefix", + protocol)); + return nullptr; + } + + private: + static std::string tolower(std::string s) { + std::transform(s.begin(), s.end(), s.begin(), + [](unsigned char c) { return std::tolower(c); } + ); + return s; + } + + static std::string extractProtocol(std::string url) { + return url.substr(0, url.find("://")); + } + + static std::string stripProtocol(std::string url) { + size_t pos = url.find("://"); + return pos == url.npos ? url : url.substr(pos + 3); + } + + std::map factories_; +}; + +} // namespace KINETO_NAMESPACE diff --git a/tb_plugins/profiling/libkineto/src/ActivityProfilerController.cpp b/tb_plugins/profiling/libkineto/src/ActivityProfilerController.cpp new file mode 100644 index 000000000..c85d41ed7 --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/ActivityProfilerController.cpp @@ -0,0 +1,246 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#include "ActivityProfilerController.h" + +#include +#include + +#include "ActivityLoggerFactory.h" +#include "ActivityTrace.h" +#include "CuptiActivityApi.h" +#ifdef HAS_ROCTRACER +#include "RoctracerActivityApi.h" +#endif +#include "ThreadUtil.h" +#include "output_json.h" +#include "output_membuf.h" + +#include "Logger.h" + +using namespace std::chrono; + +namespace KINETO_NAMESPACE { + +constexpr milliseconds kProfilerIntervalMsecs(1000); + +ActivityProfilerController::ActivityProfilerController( + ConfigLoader& configLoader, bool cpuOnly) + : configLoader_(configLoader) { +#ifdef HAS_ROCTRACER + profiler_ = std::make_unique( + RoctracerActivityApi::singleton(), cpuOnly); +#else + profiler_ = std::make_unique( + CuptiActivityApi::singleton(), cpuOnly); +#endif + configLoader_.addHandler(ConfigLoader::ConfigKind::ActivityProfiler, this); +} + +ActivityProfilerController::~ActivityProfilerController() { + configLoader_.removeHandler( + ConfigLoader::ConfigKind::ActivityProfiler, this); + if (profilerThread_) { + // signaling termination of the profiler loop + stopRunloop_ = true; + profilerThread_->join(); + delete profilerThread_; + profilerThread_ = nullptr; + } +} + +static ActivityLoggerFactory initLoggerFactory() { + ActivityLoggerFactory factory; + factory.addProtocol("file", [](const std::string& url) { + return std::unique_ptr(new ChromeTraceLogger(url)); + }); + return factory; +} + +static ActivityLoggerFactory& loggerFactory() { + static ActivityLoggerFactory factory = initLoggerFactory(); + return factory; +} + +void ActivityProfilerController::addLoggerFactory( + const std::string& protocol, ActivityLoggerFactory::FactoryFunc factory) { + loggerFactory().addProtocol(protocol, factory); +} + +static std::unique_ptr makeLogger(const Config& config) { + if (config.activitiesLogToMemory()) { + return std::make_unique(config); + } + return loggerFactory().makeLogger(config.activitiesLogUrl()); +} + +bool ActivityProfilerController::canAcceptConfig() { + return !profiler_->isActive(); +} + +void ActivityProfilerController::acceptConfig(const Config& config) { + VLOG(1) << "acceptConfig"; + if (config.activityProfilerEnabled()) { + scheduleTrace(config); + } +} + +void ActivityProfilerController::profilerLoop() { + setThreadName("Kineto Activity Profiler"); + VLOG(0) << "Entering activity profiler loop"; + + auto now = system_clock::now(); + auto next_wakeup_time = now + kProfilerIntervalMsecs; + + while (!stopRunloop_) { + now = system_clock::now(); + + while (now < next_wakeup_time) { + /* sleep override */ + std::this_thread::sleep_for(next_wakeup_time - now); + now = system_clock::now(); + } + + if (!profiler_->isActive()) { + std::lock_guard lock(asyncConfigLock_); + if (asyncRequestConfig_ + && !asyncRequestConfig_->hasProfileStartIteration()) { + // Note on now + kProfilerIntervalMsecs + // Profiler interval does not align perfectly upto startTime - warmup. Waiting until the next tick + // won't allow sufficient time for the profiler to warm up. So check if we are very close to the warmup time and trigger warmup + if (now + kProfilerIntervalMsecs + >= (asyncRequestConfig_->requestTimestamp() - asyncRequestConfig_->activitiesWarmupDuration())) { + LOG(INFO) << "Received on-demand activity trace request by " + << " profile timestamp = " + << asyncRequestConfig_-> + requestTimestamp().time_since_epoch().count(); + activateConfig(now); + } + } + } + + while (next_wakeup_time < now) { + next_wakeup_time += kProfilerIntervalMsecs; + } + + if (profiler_->isActive()) { + next_wakeup_time = profiler_->performRunLoopStep(now, next_wakeup_time); + VLOG(1) << "Profiler loop: " + << duration_cast(system_clock::now() - now).count() + << "ms"; + } + } + + VLOG(0) << "Exited activity profiling loop"; +} + +void ActivityProfilerController::step() { + int64_t currentIter = ++iterationCount_; + VLOG(0) << "Step called , iteration = " << currentIter; + + // optimization to not take the lock unless necessary + if (asyncRequestConfig_ && !profiler_->isActive()) { + std::lock_guard lock(asyncConfigLock_); + auto startIter = asyncRequestConfig_->startIterationIncludingWarmup(); + + if (asyncRequestConfig_->hasProfileStartIteration() + && currentIter >= startIter) { + LOG(INFO) << "Received on-demand activity trace request by profile" + << " start iteration = " + << asyncRequestConfig_->profileStartIteration() + << " current iteration = " << currentIter; + + if (currentIter > startIter) { + // adjust the start iteration if it is in the past + auto newProfileStart = currentIter + + asyncRequestConfig_->activitiesWarmupIterations(); + LOG(INFO) << "Start iteration updated to " << newProfileStart; + asyncRequestConfig_->setProfileStartIteration(newProfileStart); + } + activateConfig(system_clock::now()); + } + } + + if (profiler_->isActive()) { + auto now = system_clock::now(); + auto next_wakeup_time = now + kProfilerIntervalMsecs; + profiler_->performRunLoopStep(now, next_wakeup_time, currentIter); + } +} + +void ActivityProfilerController::activateConfig( + std::chrono::time_point now) { + logger_ = makeLogger(*asyncRequestConfig_); + profiler_->setLogger(logger_.get()); + profiler_->configure(*asyncRequestConfig_, now); + asyncRequestConfig_ = nullptr; +} + +void ActivityProfilerController::scheduleTrace(const Config& config) { + VLOG(1) << "scheduleTrace"; + if (profiler_->isActive()) { + LOG(ERROR) << "Ignored request - profiler busy"; + return; + } + int64_t currentIter = iterationCount_; + if (config.hasProfileStartIteration() && currentIter < 0) { + LOG(ERROR) << "Ignored profile iteration count based request as " + << "application is not updating iteration count"; + return; + } + std::lock_guard lock(asyncConfigLock_); + asyncRequestConfig_ = config.clone(); + + auto startIter = asyncRequestConfig_->startIterationIncludingWarmup(); + + if (asyncRequestConfig_->hasProfileStartIteration() + && (currentIter > startIter) + && asyncRequestConfig_->profileStartIterationRoundUp() > 0) { + auto newProfileStart + = currentIter + asyncRequestConfig_->activitiesWarmupIterations(); + // round up to nearest multiple + auto divisor = asyncRequestConfig_->profileStartIterationRoundUp(); + auto rem = newProfileStart % divisor; + newProfileStart += ((rem == 0) ? 0 : divisor - rem); + LOG(INFO) << "Rounding up profiler start iteration to : " << newProfileStart; + asyncRequestConfig_->setProfileStartIteration(newProfileStart); + } + + // start a profilerLoop() thread to handle request + if (!profilerThread_) { + profilerThread_ = + new std::thread(&ActivityProfilerController::profilerLoop, this); + } +} + +void ActivityProfilerController::prepareTrace(const Config& config) { + // Requests from ActivityProfilerApi have higher priority than + // requests from other sources (signal, daemon). + // Cancel any ongoing request and refuse new ones. + auto now = system_clock::now(); + if (profiler_->isActive()) { + LOG(WARNING) << "Cancelling current trace request in order to start " + << "higher priority synchronous request"; + if (libkineto::api().client()) { + libkineto::api().client()->stop(); + } + profiler_->stopTrace(now); + profiler_->reset(); + } + + profiler_->configure(config, now); +} + +std::unique_ptr ActivityProfilerController::stopTrace() { + profiler_->stopTrace(std::chrono::system_clock::now()); + auto logger = std::make_unique(profiler_->config()); + profiler_->processTrace(*logger); + profiler_->reset(); + return std::make_unique(std::move(logger), loggerFactory()); +} + +void ActivityProfilerController::addMetadata( + const std::string& key, const std::string& value) { + profiler_->addMetadata(key, value); +} + +} // namespace KINETO_NAMESPACE diff --git a/tb_plugins/profiling/libkineto/src/ActivityProfilerController.h b/tb_plugins/profiling/libkineto/src/ActivityProfilerController.h new file mode 100644 index 000000000..415f107cb --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/ActivityProfilerController.h @@ -0,0 +1,84 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#pragma once + +#include +#include +#include +#include +#include + +#include "ActivityLoggerFactory.h" +#include "CuptiActivityProfiler.h" +#include "ActivityProfilerInterface.h" +#include "ActivityTraceInterface.h" +#include "ConfigLoader.h" +#include "CuptiActivityApi.h" + +namespace KINETO_NAMESPACE { + +class Config; + +class ActivityProfilerController : public ConfigLoader::ConfigHandler { + public: + explicit ActivityProfilerController(ConfigLoader& configLoader, bool cpuOnly); + ActivityProfilerController(const ActivityProfilerController&) = delete; + ActivityProfilerController& operator=(const ActivityProfilerController&) = + delete; + + ~ActivityProfilerController(); + + static void addLoggerFactory( + const std::string& protocol, + ActivityLoggerFactory::FactoryFunc factory); + + bool canAcceptConfig() override; + void acceptConfig(const Config& config) override; + + void scheduleTrace(const Config& config); + + void prepareTrace(const Config& config); + + void startTrace() { + profiler_->startTrace(std::chrono::system_clock::now()); + } + + void step(); + + std::unique_ptr stopTrace(); + + bool isActive() { + return profiler_->isActive(); + } + + void transferCpuTrace( + std::unique_ptr cpuTrace) { + return profiler_->transferCpuTrace(std::move(cpuTrace)); + } + + void recordThreadInfo() { + profiler_->recordThreadInfo(); + } + + void addChildActivityProfiler( + std::unique_ptr profiler) { + profiler_->addChildActivityProfiler(std::move(profiler)); + } + + void addMetadata(const std::string& key, const std::string& value); + + private: + void profilerLoop(); + void activateConfig(std::chrono::time_point now); + + std::unique_ptr asyncRequestConfig_; + std::mutex asyncConfigLock_; + std::unique_ptr profiler_; + std::unique_ptr logger_; + std::thread* profilerThread_{nullptr}; + std::atomic_bool stopRunloop_{false}; + std::atomic iterationCount_{-1}; + ConfigLoader& configLoader_; +}; + +} // namespace KINETO_NAMESPACE diff --git a/tb_plugins/profiling/libkineto/src/ActivityProfilerProxy.cpp b/tb_plugins/profiling/libkineto/src/ActivityProfilerProxy.cpp new file mode 100644 index 000000000..b2d36b7b3 --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/ActivityProfilerProxy.cpp @@ -0,0 +1,119 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#include "ActivityProfilerProxy.h" + +#include "ActivityProfilerController.h" +#include "Config.h" +#include "CuptiActivityApi.h" +#include "Logger.h" +#include + +namespace KINETO_NAMESPACE { + +ActivityProfilerProxy::ActivityProfilerProxy( + bool cpuOnly, ConfigLoader& configLoader) + : cpuOnly_(cpuOnly), configLoader_(configLoader) { +} + +ActivityProfilerProxy::~ActivityProfilerProxy() { + delete controller_; +}; + +void ActivityProfilerProxy::init() { + if (!controller_) { + controller_ = new ActivityProfilerController(configLoader_, cpuOnly_); + } +} + +void ActivityProfilerProxy::scheduleTrace(const std::string& configStr) { + Config config; + config.parse(configStr); + controller_->scheduleTrace(config); +} + +void ActivityProfilerProxy::scheduleTrace(const Config& config) { + controller_->scheduleTrace(config); +} + +void ActivityProfilerProxy::prepareTrace( + const std::set& activityTypes, + const std::string& configStr) { + Config config; + bool validate_required = true; + + // allow user provided config to override default options + if (!configStr.empty()) { + if (!config.parse(configStr)) { + LOG(WARNING) << "Failed to parse config : " << configStr; + } + // parse also runs validate + validate_required = false; + } + + config.setClientDefaults(); + config.setSelectedActivityTypes(activityTypes); + + if (validate_required) { + config.validate(std::chrono::system_clock::now()); + } + + controller_->prepareTrace(config); +} + +void ActivityProfilerProxy::startTrace() { + controller_->startTrace(); +} + +std::unique_ptr +ActivityProfilerProxy::stopTrace() { + return controller_->stopTrace(); +} + +void ActivityProfilerProxy::step() { + controller_->step(); +} + +bool ActivityProfilerProxy::isActive() { + return controller_->isActive(); +} + +void ActivityProfilerProxy::pushCorrelationId(uint64_t id) { + CuptiActivityApi::pushCorrelationID(id, + CuptiActivityApi::CorrelationFlowType::Default); +} + +void ActivityProfilerProxy::popCorrelationId() { + CuptiActivityApi::popCorrelationID( + CuptiActivityApi::CorrelationFlowType::Default); +} + +void ActivityProfilerProxy::pushUserCorrelationId(uint64_t id) { + CuptiActivityApi::pushCorrelationID(id, + CuptiActivityApi::CorrelationFlowType::User); +} + +void ActivityProfilerProxy::popUserCorrelationId() { + CuptiActivityApi::popCorrelationID( + CuptiActivityApi::CorrelationFlowType::User); +} + +void ActivityProfilerProxy::transferCpuTrace( + std::unique_ptr traceBuffer) { + controller_->transferCpuTrace(std::move(traceBuffer)); +} + +void ActivityProfilerProxy::addMetadata( + const std::string& key, const std::string& value) { + controller_->addMetadata(key, value); +} + +void ActivityProfilerProxy::recordThreadInfo() { + controller_->recordThreadInfo(); +} + +void ActivityProfilerProxy::addChildActivityProfiler( + std::unique_ptr profiler) { + controller_->addChildActivityProfiler(std::move(profiler)); +} + +} // namespace libkineto diff --git a/tb_plugins/profiling/libkineto/src/ActivityProfilerProxy.h b/tb_plugins/profiling/libkineto/src/ActivityProfilerProxy.h new file mode 100644 index 000000000..b5cf84b2f --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/ActivityProfilerProxy.h @@ -0,0 +1,73 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#pragma once + +#include "ActivityProfilerInterface.h" + +#include +#include +#include + +#include "ActivityType.h" +#include "ITraceActivity.h" + +namespace libkineto { + // previous declaration is struct so this one must be too. + struct CpuTraceBuffer; +} + +namespace KINETO_NAMESPACE { + +using namespace libkineto; + +class ActivityProfilerController; +class Config; +class ConfigLoader; + +class ActivityProfilerProxy : public ActivityProfilerInterface { + + public: + ActivityProfilerProxy(bool cpuOnly, ConfigLoader& configLoader); + ~ActivityProfilerProxy() override; + + void init() override; + bool isInitialized() override { + return controller_ != nullptr; + } + + bool isActive() override; + + void recordThreadInfo() override; + + void scheduleTrace(const std::string& configStr) override; + void scheduleTrace(const Config& config); + + void prepareTrace( + const std::set& activityTypes, + const std::string& configStr = "") override; + + void startTrace() override; + void step() override; + std::unique_ptr stopTrace() override; + + void pushCorrelationId(uint64_t id) override; + void popCorrelationId() override; + + void pushUserCorrelationId(uint64_t id) override; + void popUserCorrelationId() override; + + void transferCpuTrace( + std::unique_ptr traceBuffer) override; + + void addMetadata(const std::string& key, const std::string& value) override; + + virtual void addChildActivityProfiler( + std::unique_ptr profiler) override; + + private: + bool cpuOnly_{true}; + ConfigLoader& configLoader_; + ActivityProfilerController* controller_{nullptr}; +}; + +} // namespace libkineto diff --git a/tb_plugins/profiling/libkineto/src/ActivityTrace.h b/tb_plugins/profiling/libkineto/src/ActivityTrace.h new file mode 100644 index 000000000..0be76af08 --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/ActivityTrace.h @@ -0,0 +1,45 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#pragma once + +#include +#include + +#include "ActivityLoggerFactory.h" +#include "ActivityTraceInterface.h" +#include "output_json.h" +#include "output_membuf.h" + +namespace libkineto { + +class ActivityTrace : public ActivityTraceInterface { + public: + ActivityTrace( + std::unique_ptr tmpLogger, + const ActivityLoggerFactory& factory) + : memLogger_(std::move(tmpLogger)), + loggerFactory_(factory) { + } + + const std::vector* activities() override { + return memLogger_->traceActivities(); + }; + + void save(const std::string& url) override { + std::string prefix; + // if no protocol is specified, default to file + if (url.find("://") == url.npos) { + prefix = "file://"; + } + memLogger_->log(*loggerFactory_.makeLogger(prefix + url)); + }; + + private: + // Activities are logged into a buffer + std::unique_ptr memLogger_; + + // Alternative logger used by save() if protocol prefix is specified + const ActivityLoggerFactory& loggerFactory_; +}; + +} // namespace libkineto diff --git a/tb_plugins/profiling/libkineto/src/ActivityType.cpp b/tb_plugins/profiling/libkineto/src/ActivityType.cpp new file mode 100644 index 000000000..18856b723 --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/ActivityType.cpp @@ -0,0 +1,58 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#include "ActivityType.h" + +#include + +namespace libkineto { + +struct ActivityTypeName { + const char* name; + ActivityType type; +}; + +static constexpr std::array map{{ + {"cpu_op", ActivityType::CPU_OP}, + {"user_annotation", ActivityType::USER_ANNOTATION}, + {"gpu_user_Annotation", ActivityType::GPU_USER_ANNOTATION}, + {"gpu_memcpy", ActivityType::GPU_MEMCPY}, + {"gpu_memset", ActivityType::GPU_MEMSET}, + {"kernel", ActivityType::CONCURRENT_KERNEL}, + {"external_correlation", ActivityType::EXTERNAL_CORRELATION}, + {"cuda_runtime", ActivityType::CUDA_RUNTIME}, + {"cuda_profiler_range", ActivityType::CUDA_PROFILER_RANGE}, + {"glow_runtime", ActivityType::GLOW_RUNTIME}, + {"cpu_instant_event", ActivityType::CPU_INSTANT_EVENT}, + {"python_function", ActivityType::PYTHON_FUNCTION}, + {"overhead", ActivityType::OVERHEAD}, + {"ENUM_COUNT", ActivityType::ENUM_COUNT} +}}; + +static constexpr bool matchingOrder(int idx = 0) { + return map[idx].type == ActivityType::ENUM_COUNT || + ((idx == (int) map[idx].type) && matchingOrder(idx + 1)); +} +static_assert(matchingOrder(), "ActivityTypeName map is out of order"); + +const char* toString(ActivityType t) { + return map[(int)t].name; +} + +ActivityType toActivityType(const std::string& str) { + for (int i = 0; i < activityTypeCount; i++) { + if (str == map[i].name) { + return map[i].type; + } + } + throw std::invalid_argument(fmt::format("Invalid activity type: {}", str)); +} + +const std::array activityTypes() { + std::array res; + for (int i = 0; i < activityTypeCount; i++) { + res[i] = map[i].type; + } + return res; +} + +} // namespace libkineto diff --git a/tb_plugins/profiling/libkineto/src/Config.cpp b/tb_plugins/profiling/libkineto/src/Config.cpp new file mode 100644 index 000000000..95538840f --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/Config.cpp @@ -0,0 +1,473 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#include "Config.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "Logger.h" +#include "ThreadUtil.h" + +using namespace std::chrono; + +using std::string; +using std::vector; + +namespace KINETO_NAMESPACE { + +constexpr milliseconds kDefaultSamplePeriodMsecs(1000); +constexpr milliseconds kDefaultMultiplexPeriodMsecs(1000); +constexpr milliseconds kDefaultActivitiesProfileDurationMSecs(500); +constexpr int kDefaultActivitiesMaxGpuBufferSize(128 * 1024 * 1024); +constexpr seconds kDefaultActivitiesWarmupDurationSecs(5); +constexpr seconds kDefaultBufferUntilWarmup(10); +constexpr seconds kDefaultReportPeriodSecs(1); +constexpr int kDefaultSamplesPerReport(1); +constexpr int kDefaultMaxEventProfilersPerGpu(1); +constexpr int kDefaultEventProfilerHearbeatMonitorPeriod(0); +constexpr seconds kMaxRequestAge(10); + +// Event Profiler +constexpr char kEventsKey[] = "EVENTS"; +constexpr char kMetricsKey[] = "METRICS"; +constexpr char kSamplePeriodKey[] = "SAMPLE_PERIOD_MSECS"; +constexpr char kMultiplexPeriodKey[] = "MULTIPLEX_PERIOD_MSECS"; +constexpr char kReportPeriodKey[] = "REPORT_PERIOD_SECS"; +constexpr char kSamplesPerReportKey[] = "SAMPLES_PER_REPORT"; +constexpr char kEventsLogFileKey[] = "EVENTS_LOG_FILE"; +constexpr char kEventsEnabledDevicesKey[] = "EVENTS_ENABLED_DEVICES"; +constexpr char kOnDemandDurationKey[] = "EVENTS_DURATION_SECS"; +constexpr char kMaxEventProfilersPerGpuKey[] = "MAX_EVENT_PROFILERS_PER_GPU"; +constexpr char kHeartbeatMonitorPeriodKey[] = + "EVENTS_HEARTBEAT_MONITOR_PERIOD_SECS"; + +// Activity Profiler +constexpr char kActivitiesEnabledKey[] = "ACTIVITIES_ENABLED"; +constexpr char kActivityTypesKey[] = "ACTIVITY_TYPES"; +constexpr char kActivitiesLogFileKey[] = "ACTIVITIES_LOG_FILE"; +constexpr char kActivitiesDurationKey[] = "ACTIVITIES_DURATION_SECS"; +constexpr char kActivitiesDurationMsecsKey[] = "ACTIVITIES_DURATION_MSECS"; +constexpr char kActivitiesWarmupDurationSecsKey[] = "ACTIVITIES_WARMUP_PERIOD_SECS"; +constexpr char kActivitiesMaxGpuBufferSizeKey[] = + "ACTIVITIES_MAX_GPU_BUFFER_SIZE_MB"; + +// Client Interface +constexpr char kClientInterfaceEnableOpInputsCollection[] = "CLIENT_INTERFACE_ENABLE_OP_INPUTS_COLLECTION"; + +constexpr char kActivitiesWarmupIterationsKey[] = "ACTIVITIES_WARMUP_ITERATIONS"; +constexpr char kActivitiesIterationsKey[] = "ACTIVITIES_ITERATIONS"; +// Common + +// Client-side timestamp used for synchronized start across hosts for +// distributed workloads. +// Specified in milliseconds Unix time (milliseconds since epoch). +// To use, compute a future timestamp as follows: +// * C++: + duration_cast( +// system_clock::now().time_since_epoch()).count() +// * Python: + int(time.time() * 1000) +// * Bash: $(( + $(date +%s%3N))) +// If used for a tracing request, timestamp must be far enough in the future +// to accommodate ACTIVITIES_WARMUP_PERIOD_SECS as well as any delays in +// propagating the request to the profiler. +// If the request can not be honored, it is up to the profilers to report +// an error somehow - no checks are done at config parse time. +// Note PROFILE_START_ITERATION has higher precedence +constexpr char kProfileStartTimeKey[] = "PROFILE_START_TIME"; +// DEPRECATED - USE PROFILE_START_TIME instead +constexpr char kRequestTimestampKey[] = "REQUEST_TIMESTAMP"; + +// Alternatively if the application supports reporting iterations +// start the profile at specific iteration. If the iteration count +// is >= this value the profile is started immediately. +// A value >= 0 is valid for this config option to take effect. +// Note PROFILE_START_ITERATION will take precedence over PROFILE_START_TIME. +constexpr char kProfileStartIterationKey[] = "PROFILE_START_ITERATION"; + +// Users can also start the profile on an integer multiple of the config +// value PROFILE_START_ITERATION_ROUNDUP. This knob behaves similar to +// PROFILE_START_ITERATION but instead of saying : "start collection trace on +// iteration 500", one can configure it to "start collecting trace on the next +// 100th iteration". +// +// For example, +// PROFILE_START_ITERATION_ROUNDUP = 1000, and the current iteration is 2010 +// The profile will then be collected on the next multiple of 1000 ie. 3000 +// Note PROFILE_START_ITERATION_ROUNDUP will also take precedence over +// PROFILE_START_TIME. +constexpr char kProfileStartIterationRoundUpKey[] + = "PROFILE_START_ITERATION_ROUNDUP"; + +// Enable on-demand trigger via kill -USR2 +// When triggered in this way, /tmp/libkineto.conf will be used as config. +constexpr char kEnableSigUsr2Key[] = "ENABLE_SIGUSR2"; + +// Enable communication through IPC Fabric +// and disable thrift communication with dynolog daemon +constexpr char kEnableIpcFabricKey[] = "ENABLE_IPC_FABRIC"; + +// Verbose log level +// The actual glog is not used and --v and --vmodule has no effect. +// Instead set the verbose level and modules in the config file. +constexpr char kLogVerboseLevelKey[] = "VERBOSE_LOG_LEVEL"; +// By default, all modules will log verbose messages >= verboseLogLevel. +// But to reduce noise we can specify one or more modules of interest. +// A module is a C/C++ object file (source file name), +// Example argument: ActivityProfiler.cpp,output_json.cpp +constexpr char kLogVerboseModulesKey[] = "VERBOSE_LOG_MODULES"; + +// Max devices supported on any system +constexpr uint8_t kMaxDevices = 8; + +namespace { + +struct FactoryMap { + + void addFactory( + std::string name, + std::function factory) { + std::lock_guard lock(lock_); + factories_[name] = factory; + } + + void addFeatureConfigs(Config& cfg) { + std::lock_guard lock(lock_); + for (const auto& p : factories_) { + cfg.addFeature(p.first, p.second(cfg)); + } + } + +// Config factories are shared between objects and since +// config objects can be created by multiple threads, we need a lock. + std::mutex lock_; + std::map> factories_; +}; + +std::shared_ptr configFactories() { + // Ensure this is safe to call during shutdown, even as static + // destructors are invoked. Once factories destructor has been + // invoked, weak_ptr.lock() will return nullptr. + // But calls before that point will have a valid shared_ptr, + // delaying destruction of the underlying FactoryMap. + static auto factories = std::make_shared(); + static std::weak_ptr weak_ptr = factories; + return weak_ptr.lock(); +} + +} // namespace + +void Config::addConfigFactory( + std::string name, + std::function factory) { + auto factories = configFactories(); + if (factories) { + factories->addFactory(name, factory); + } +} + +static string defaultTraceFileName() { + return fmt::format("/tmp/libkineto_activities_{}.json", processId()); +} + +Config::Config() + : verboseLogLevel_(-1), + samplePeriod_(kDefaultSamplePeriodMsecs), + reportPeriod_(duration_cast(kDefaultReportPeriodSecs)), + samplesPerReport_(kDefaultSamplesPerReport), + eventProfilerOnDemandDuration_(seconds(0)), + eventProfilerMaxInstancesPerGpu_(kDefaultMaxEventProfilersPerGpu), + eventProfilerHeartbeatMonitorPeriod_( + kDefaultEventProfilerHearbeatMonitorPeriod), + multiplexPeriod_(kDefaultMultiplexPeriodMsecs), + activityProfilerEnabled_(true), + activitiesLogFile_(defaultTraceFileName()), + activitiesLogUrl_(fmt::format("file://{}", activitiesLogFile_)), + activitiesMaxGpuBufferSize_(kDefaultActivitiesMaxGpuBufferSize), + activitiesWarmupDuration_(kDefaultActivitiesWarmupDurationSecs), + activitiesWarmupIterations_(0), + activitiesDuration_(kDefaultActivitiesProfileDurationMSecs), + activitiesRunIterations_(0), + activitiesOnDemandTimestamp_(milliseconds(0)), + profileStartTime_(milliseconds(0)), + profileStartIteration_(-1), + profileStartIterationRoundUp_(-1), + requestTimestamp_(milliseconds(0)), + enableSigUsr2_(false), + enableIpcFabric_(false) { + auto factories = configFactories(); + if (factories) { + factories->addFeatureConfigs(*this); + } +} + +uint8_t Config::createDeviceMask(const string& val) { + uint8_t res = 0; + for (const auto& d : splitAndTrim(val, ',')) { + res |= 1 << toIntRange(d, 0, kMaxDevices - 1); + } + return res; +} + +const seconds Config::maxRequestAge() const { + return kMaxRequestAge; +} + +static std::string getTimeStr(time_point t) { + std::time_t t_c = system_clock::to_time_t(t); + return fmt::format("{:%H:%M:%S}", fmt::localtime(t_c)); +} + +static time_point handleRequestTimestamp(int64_t ms) { + auto t = time_point(milliseconds(ms)); + auto now = system_clock::now(); + if (t > now) { + throw std::invalid_argument(fmt::format( + "Invalid {}: {} - time is in future", + kRequestTimestampKey, + getTimeStr(t))); + } else if ((now - t) > kMaxRequestAge) { + throw std::invalid_argument(fmt::format( + "Invalid {}: {} - time is more than {}s in the past", + kRequestTimestampKey, + getTimeStr(t), + kMaxRequestAge.count())); + } + return t; +} + +void Config::setActivityTypes( + const std::vector& selected_activities) { + selectedActivityTypes_.clear(); + if (selected_activities.size() > 0) { + for (const auto& activity : selected_activities) { + if (activity == "") { + continue; + } + selectedActivityTypes_.insert(toActivityType(activity)); + } + } +} + +bool Config::handleOption(const std::string& name, std::string& val) { + // Event Profiler + if (!name.compare(kEventsKey)) { + vector event_names = splitAndTrim(val, ','); + eventNames_.insert(event_names.begin(), event_names.end()); + } else if (!name.compare(kMetricsKey)) { + vector metric_names = splitAndTrim(val, ','); + metricNames_.insert(metric_names.begin(), metric_names.end()); + } else if (!name.compare(kSamplePeriodKey)) { + samplePeriod_ = milliseconds(toInt32(val)); + } else if (!name.compare(kMultiplexPeriodKey)) { + multiplexPeriod_ = milliseconds(toInt32(val)); + } else if (!name.compare(kReportPeriodKey)) { + setReportPeriod(seconds(toInt32(val))); + } else if (!name.compare(kSamplesPerReportKey)) { + samplesPerReport_ = toInt32(val); + } else if (!name.compare(kEventsLogFileKey)) { + eventLogFile_ = val; + } else if (!name.compare(kEventsEnabledDevicesKey)) { + eventProfilerDeviceMask_ = createDeviceMask(val); + } else if (!name.compare(kOnDemandDurationKey)) { + eventProfilerOnDemandDuration_ = seconds(toInt32(val)); + eventProfilerOnDemandTimestamp_ = timestamp(); + } else if (!name.compare(kMaxEventProfilersPerGpuKey)) { + eventProfilerMaxInstancesPerGpu_ = toInt32(val); + } else if (!name.compare(kHeartbeatMonitorPeriodKey)) { + eventProfilerHeartbeatMonitorPeriod_ = seconds(toInt32(val)); + } + + // Activity Profiler + else if (!name.compare(kActivitiesDurationKey)) { + activitiesDuration_ = + duration_cast(seconds(toInt32(val))); + activitiesOnDemandTimestamp_ = timestamp(); + } else if (!name.compare(kActivityTypesKey)) { + vector activity_types = splitAndTrim(toLower(val), ','); + setActivityTypes(activity_types); + } else if (!name.compare(kActivitiesDurationMsecsKey)) { + activitiesDuration_ = milliseconds(toInt32(val)); + activitiesOnDemandTimestamp_ = timestamp(); + } else if (!name.compare(kActivitiesIterationsKey)) { + activitiesRunIterations_ = toInt32(val); + activitiesOnDemandTimestamp_ = timestamp(); + } else if (!name.compare(kLogVerboseLevelKey)) { + verboseLogLevel_ = toInt32(val); + } else if (!name.compare(kLogVerboseModulesKey)) { + verboseLogModules_ = splitAndTrim(val, ','); + } else if (!name.compare(kActivitiesEnabledKey)) { + activityProfilerEnabled_ = toBool(val); + } else if (!name.compare(kActivitiesLogFileKey)) { + activitiesLogFile_ = val; + activitiesLogUrl_ = fmt::format("file://{}", val); + activitiesOnDemandTimestamp_ = timestamp(); + } else if (!name.compare(kActivitiesMaxGpuBufferSizeKey)) { + activitiesMaxGpuBufferSize_ = toInt32(val) * 1024 * 1024; + } else if (!name.compare(kActivitiesWarmupDurationSecsKey)) { + activitiesWarmupDuration_ = seconds(toInt32(val)); + } else if (!name.compare(kActivitiesWarmupIterationsKey)) { + activitiesWarmupIterations_ = toInt32(val); + } + + // Client Interface + else if (!name.compare(kClientInterfaceEnableOpInputsCollection)) { + enableOpInputsCollection_ = toBool(val); + } + + // Common + else if (!name.compare(kRequestTimestampKey)) { + VLOG(0) << kRequestTimestampKey + << " has been deprecated - please use " + << kProfileStartTimeKey; + requestTimestamp_ = handleRequestTimestamp(toInt64(val)); + } else if (!name.compare(kProfileStartTimeKey)) { + profileStartTime_ = + time_point(milliseconds(toInt64(val))); + } else if (!name.compare(kProfileStartIterationKey)) { + profileStartIteration_ = toInt32(val); + } else if (!name.compare(kProfileStartIterationRoundUpKey)) { + profileStartIterationRoundUp_ = toInt32(val); + } else if (!name.compare(kEnableSigUsr2Key)) { + enableSigUsr2_ = toBool(val); + } else if (!name.compare(kEnableIpcFabricKey)) { + enableIpcFabric_ = toBool(val); + } else { + return false; + } + return true; +} + +std::chrono::milliseconds Config::activitiesDurationDefault() const { + return kDefaultActivitiesProfileDurationMSecs; +}; + +void Config::updateActivityProfilerRequestReceivedTime() { + activitiesOnDemandTimestamp_ = system_clock::now(); +} + +void Config::setClientDefaults() { + AbstractConfig::setClientDefaults(); + activitiesLogToMemory_ = true; +} + +void Config::validate( + const time_point& fallbackProfileStartTime) { + if (samplePeriod_.count() == 0) { + LOG(WARNING) << "Sample period must be greater than 0, setting to 1ms"; + samplePeriod_ = milliseconds(1); + } + + if (multiplexPeriod_ < samplePeriod_) { + LOG(WARNING) << "Multiplex period can not be smaller " + << "than sample period"; + LOG(WARNING) << "Setting multiplex period to " << samplePeriod_.count() + << "ms"; + multiplexPeriod_ = samplePeriod_; + } + + if ((multiplexPeriod_ % samplePeriod_).count() != 0) { + LOG(WARNING) << "Multiplex period must be a " + << "multiple of sample period"; + multiplexPeriod_ = alignUp(multiplexPeriod_, samplePeriod_); + LOG(WARNING) << "Setting multiplex period to " << multiplexPeriod_.count() + << "ms"; + } + + if ((reportPeriod_ % multiplexPeriod_).count() != 0 || + reportPeriod_.count() == 0) { + LOG(WARNING) << "Report period must be a " + << "multiple of multiplex period"; + reportPeriod_ = alignUp(reportPeriod_, multiplexPeriod_); + LOG(WARNING) << "Setting report period to " << reportPeriod_.count() + << "ms"; + } + + if (samplesPerReport_ < 1) { + LOG(WARNING) << "Samples per report must be in the range " + << "[1, report period / sample period]"; + LOG(WARNING) << "Setting samples per report to 1"; + samplesPerReport_ = 1; + } + + int max_samples_per_report = reportPeriod_ / samplePeriod_; + if (samplesPerReport_ > max_samples_per_report) { + LOG(WARNING) << "Samples per report must be in the range " + << "[1, report period / sample period] ([1, " + << reportPeriod_.count() << "ms / " << samplePeriod_.count() + << "ms = " << max_samples_per_report << "])"; + LOG(WARNING) << "Setting samples per report to " << max_samples_per_report; + samplesPerReport_ = max_samples_per_report; + } + + if (!hasProfileStartTime()) { + VLOG(0) + << "No explicit timestamp has been set. " + << "Defaulting it to now + activitiesWarmupDuration with buffer."; + profileStartTime_ = fallbackProfileStartTime + + activitiesWarmupDuration() + kDefaultBufferUntilWarmup; + } + + if (profileStartIterationRoundUp_ == 0) { + // setting to 0 will mess up modulo arithmetic, set it to -1 so it has no effect + LOG(WARNING) << "Profiler start iteration round up should be >= 1."; + profileStartIterationRoundUp_ = -1; + } + + if (profileStartIterationRoundUp_ > 0 && !hasProfileStartIteration()) { + VLOG(0) << "Setting profiler start iteration to 0 so this config is " + << "triggered via iteration count."; + profileStartIteration_ = 0; + } + + if (selectedActivityTypes_.size() == 0) { + selectDefaultActivityTypes(); + } +} + +void Config::setReportPeriod(milliseconds msecs) { + reportPeriod_ = msecs; +} + +void Config::printActivityProfilerConfig(std::ostream& s) const { + s << "Log file: " << activitiesLogFile() << std::endl; + if (hasProfileStartIteration()) { + s << "Trace start Iteration: " << profileStartIteration() << std::endl; + s << "Trace warmup Iterations: " << activitiesWarmupIterations() << std::endl; + s << "Trace profile Iterations: " << activitiesRunIterations() << std::endl; + if (profileStartIterationRoundUp() > 0) { + s << "Trace start iteration roundup : " << profileStartIterationRoundUp() + << std::endl; + } + } else if (hasProfileStartTime()) { + std::time_t t_c = system_clock::to_time_t(requestTimestamp()); + LOG(INFO) << "Trace start time: " + << fmt::format("{:%Y-%m-%d %H:%M:%S}", fmt::localtime(t_c)); + s << "Trace duration: " << activitiesDuration().count() << "ms" + << std::endl; + s << "Warmup duration: " << activitiesWarmupDuration().count() << "s" + << std::endl; + } + + s << "Max GPU buffer size: " << activitiesMaxGpuBufferSize() / 1024 / 1024 + << "MB" << std::endl; + + std::vector activities; + for (const auto& activity : selectedActivityTypes_) { + activities.push_back(toString(activity)); + } + s << "Enabled activities: " + << fmt::format("{}", fmt::join(activities, ",")) << std::endl; + + AbstractConfig::printActivityProfilerConfig(s); +} + +} // namespace KINETO_NAMESPACE diff --git a/tb_plugins/profiling/libkineto/src/ConfigLoader.cpp b/tb_plugins/profiling/libkineto/src/ConfigLoader.cpp new file mode 100644 index 000000000..4080b678d --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/ConfigLoader.cpp @@ -0,0 +1,300 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#include "ConfigLoader.h" + +#ifdef __linux__ +#include +#endif + +#include +#include +#include +#include +#include + +#include "DaemonConfigLoader.h" + +#include "Logger.h" + +using namespace std::chrono; +using std::string; + +namespace KINETO_NAMESPACE { + +using namespace libkineto; + +constexpr char kConfigFileEnvVar[] = "KINETO_CONFIG"; +#ifdef __linux__ +constexpr char kConfigFile[] = "/etc/libkineto.conf"; +constexpr char kOnDemandConfigFile[] = "/tmp/libkineto.conf"; +#else +constexpr char kConfigFile[] = "libkineto.conf"; +constexpr char kOnDemandConfigFile[] = "libkineto.conf"; +#endif + +constexpr std::chrono::seconds kConfigUpdateIntervalSecs(300); +constexpr std::chrono::seconds kOnDemandConfigUpdateIntervalSecs(5); + +#ifdef __linux__ +static struct sigaction originalUsr2Handler = {}; +#endif + +// Use SIGUSR2 to initiate profiling. +// Look for an on-demand config file. +// If none is found, default to base config. +// Try to not affect existing handlers +static bool hasOriginalSignalHandler() { +#ifdef __linux__ + return originalUsr2Handler.sa_handler != nullptr || + originalUsr2Handler.sa_sigaction != nullptr; +#else + return false; +#endif +} + +static void handle_signal(int signal) { +#ifdef __linux__ + if (signal == SIGUSR2) { + ConfigLoader::instance().handleOnDemandSignal(); + if (hasOriginalSignalHandler()) { + // Invoke original handler and reinstate ours + struct sigaction act; + sigaction(SIGUSR2, &originalUsr2Handler, &act); + raise(SIGUSR2); + sigaction(SIGUSR2, &act, &originalUsr2Handler); + } + } +#endif +} + +static void setupSignalHandler(bool enableSigUsr2) { +#ifdef __linux__ + if (enableSigUsr2) { + struct sigaction act = {}; + act.sa_handler = &handle_signal; + act.sa_flags = SA_NODEFER; + if (sigaction(SIGUSR2, &act, &originalUsr2Handler) < 0) { + PLOG(ERROR) << "Failed to register SIGUSR2 handler"; + } + if (originalUsr2Handler.sa_handler == &handle_signal) { + originalUsr2Handler = {}; + } + } else if (hasOriginalSignalHandler()) { + sigaction(SIGUSR2, &originalUsr2Handler, nullptr); + originalUsr2Handler = {}; + } +#endif +} + +// return an empty string if reading gets any errors. Otherwise a config string. +static std::string readConfigFromConfigFile(const char* filename) { + // Read whole file into a string. + std::ifstream file(filename); + std::string conf; + try { + conf.assign( + std::istreambuf_iterator(file), std::istreambuf_iterator()); + } catch (std::exception& e) { + VLOG(0) << "Error reading " << filename << ": " + << e.what(); + conf = ""; + } + return conf; +} + +static std::function()>& +daemonConfigLoaderFactory() { + static std::function()> factory = nullptr; + return factory; +} + +void ConfigLoader::setDaemonConfigLoaderFactory( + std::function()> factory) { + daemonConfigLoaderFactory() = factory; +} + +ConfigLoader& ConfigLoader::instance() { + static ConfigLoader config_loader; + return config_loader; +} + +// return an empty string if polling gets any errors. Otherwise a config string. +std::string ConfigLoader::readOnDemandConfigFromDaemon( + time_point now) { + if (!daemonConfigLoader_) { + return ""; + } + bool events = canHandlerAcceptConfig(ConfigKind::EventProfiler); + bool activities = canHandlerAcceptConfig(ConfigKind::ActivityProfiler); + return daemonConfigLoader_->readOnDemandConfig(events, activities); +} + +int ConfigLoader::contextCountForGpu(uint32_t device) { + if (!daemonConfigLoader_) { + // FIXME: Throw error? + return 0; + } + return daemonConfigLoader_->gpuContextCount(device); +} + +ConfigLoader::ConfigLoader() + : configUpdateIntervalSecs_(kConfigUpdateIntervalSecs), + onDemandConfigUpdateIntervalSecs_(kOnDemandConfigUpdateIntervalSecs), + stopFlag_(false), + onDemandSignal_(false) { +} + +void ConfigLoader::startThread() { + if (!updateThread_) { + // Create default base config here - at this point static initializers + // of extensions should have run and registered all config feature factories + std::lock_guard lock(configLock_); + if (!config_) { + config_ = std::make_unique(); + } + updateThread_ = + std::make_unique(&ConfigLoader::updateConfigThread, this); + } +} + +ConfigLoader::~ConfigLoader() { + if (updateThread_) { + stopFlag_ = true; + { + std::lock_guard lock(updateThreadMutex_); + updateThreadCondVar_.notify_one(); + } + updateThread_->join(); + } +#if !USE_GOOGLE_LOG + Logger::clearLoggerObservers(); +#endif // !USE_GOOGLE_LOG +} + +void ConfigLoader::handleOnDemandSignal() { + onDemandSignal_ = true; + { + std::lock_guard lock(updateThreadMutex_); + updateThreadCondVar_.notify_one(); + } +} + +const char* ConfigLoader::configFileName() { + if (!configFileName_) { + configFileName_ = getenv(kConfigFileEnvVar); + if (configFileName_ == nullptr) { + configFileName_ = kConfigFile; + } + } + return configFileName_; +} + +DaemonConfigLoader* ConfigLoader::daemonConfigLoader() { + if (!daemonConfigLoader_ && daemonConfigLoaderFactory()) { + daemonConfigLoader_ = daemonConfigLoaderFactory()(); + daemonConfigLoader_->setCommunicationFabric(config_->ipcFabricEnabled()); + } + return daemonConfigLoader_.get(); +} + +void ConfigLoader::updateBaseConfig() { + // First try reading local config file + // If that fails, read from daemon + // TODO: Invert these once daemon path fully rolled out + std::string config_str = readConfigFromConfigFile(configFileName()); + if (config_str.empty() && daemonConfigLoader()) { + // If local config file was not successfully loaded (e.g. not found) + // then try the daemon + config_str = daemonConfigLoader()->readBaseConfig(); + } + if (config_str != config_->source()) { + std::lock_guard lock(configLock_); + config_ = std::make_unique(); + config_->parse(config_str); + if (daemonConfigLoader()) { + daemonConfigLoader()->setCommunicationFabric(config_->ipcFabricEnabled()); + } + setupSignalHandler(config_->sigUsr2Enabled()); + SET_LOG_VERBOSITY_LEVEL( + config_->verboseLogLevel(), + config_->verboseLogModules()); + VLOG(0) << "Detected base config change"; + } +} + +void ConfigLoader::configureFromSignal( + time_point now, + Config& config) { + LOG(INFO) << "Received on-demand profiling signal, " + << "reading config from " << kOnDemandConfigFile; + // Reset start time to 0 in order to compute new default start time + const std::string config_str = "PROFILE_START_TIME=0\n" + + readConfigFromConfigFile(kOnDemandConfigFile); + config.parse(config_str); + config.setSignalDefaults(); + notifyHandlers(config); +} + +void ConfigLoader::configureFromDaemon( + time_point now, + Config& config) { + const std::string config_str = readOnDemandConfigFromDaemon(now); + if (config_str.empty()) { + return; + } + + LOG(INFO) << "Received config from dyno:\n" << config_str; + config.parse(config_str); + notifyHandlers(config); +} + +void ConfigLoader::updateConfigThread() { + auto now = system_clock::now(); + auto next_config_load_time = now; + auto next_on_demand_load_time = now + onDemandConfigUpdateIntervalSecs_; + seconds interval = configUpdateIntervalSecs_; + if (interval > onDemandConfigUpdateIntervalSecs_) { + interval = onDemandConfigUpdateIntervalSecs_; + } + auto onDemandConfig = std::make_unique(); + + // This can potentially sleep for long periods of time, so allow + // the desctructor to wake it to avoid a 5-minute long destruct period. + for (;;) { + { + std::unique_lock lock(updateThreadMutex_); + updateThreadCondVar_.wait_for(lock, interval); + } + if (stopFlag_) { + break; + } + now = system_clock::now(); + if (now > next_config_load_time) { + updateBaseConfig(); + next_config_load_time = now + configUpdateIntervalSecs_; + } + if (onDemandSignal_.exchange(false)) { + onDemandConfig = config_->clone(); + configureFromSignal(now, *onDemandConfig); + } else if (now > next_on_demand_load_time) { + onDemandConfig = std::make_unique(); + configureFromDaemon(now, *onDemandConfig); + next_on_demand_load_time = now + onDemandConfigUpdateIntervalSecs_; + } + if (onDemandConfig->verboseLogLevel() >= 0) { + LOG(INFO) << "Setting verbose level to " + << onDemandConfig->verboseLogLevel() + << " from on-demand config"; + SET_LOG_VERBOSITY_LEVEL( + onDemandConfig->verboseLogLevel(), + onDemandConfig->verboseLogModules()); + } + } +} + +bool ConfigLoader::hasNewConfig(const Config& oldConfig) { + std::lock_guard lock(configLock_); + return config_->timestamp() > oldConfig.timestamp(); +} + +} // namespace KINETO_NAMESPACE diff --git a/tb_plugins/profiling/libkineto/src/ConfigLoader.h b/tb_plugins/profiling/libkineto/src/ConfigLoader.h new file mode 100644 index 000000000..4ce3468e4 --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/ConfigLoader.h @@ -0,0 +1,147 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "Config.h" + +// TODO(T90238193) +// @lint-ignore-every CLANGTIDY facebook-hte-RelativeInclude +#include "ILoggerObserver.h" + +namespace libkineto { + class LibkinetoApi; +} + +namespace KINETO_NAMESPACE { + +using namespace libkineto; +class DaemonConfigLoader; + +class ConfigLoader { + public: + + static ConfigLoader& instance(); + + enum ConfigKind { + ActivityProfiler = 0, + EventProfiler, + NumConfigKinds + }; + + struct ConfigHandler { + virtual ~ConfigHandler() {} + virtual bool canAcceptConfig() = 0; + virtual void acceptConfig(const Config& cfg) = 0; + }; + + void addHandler(ConfigKind kind, ConfigHandler* handler) { + std::lock_guard lock(updateThreadMutex_); + handlers_[kind].push_back(handler); + startThread(); + } + + void removeHandler(ConfigKind kind, ConfigHandler* handler) { + std::lock_guard lock(updateThreadMutex_); + auto it = std::find( + handlers_[kind].begin(), handlers_[kind].end(), handler); + if (it != handlers_[kind].end()) { + handlers_[kind].erase(it); + } + } + + void notifyHandlers(const Config& cfg) { + std::lock_guard lock(updateThreadMutex_); + for (auto& key_val : handlers_) { + for (ConfigHandler* handler : key_val.second) { + handler->acceptConfig(cfg); + } + } + } + + bool canHandlerAcceptConfig(ConfigKind kind) { + std::lock_guard lock(updateThreadMutex_); + for (ConfigHandler* handler : handlers_[kind]) { + if (!handler->canAcceptConfig()) { + return false; + } + } + return true; + } + + void initBaseConfig() { + bool init = false; + { + std::lock_guard lock(configLock_); + init = !config_ || config_->source().empty(); + } + if (init) { + updateBaseConfig(); + } + } + + inline std::unique_ptr getConfigCopy() { + std::lock_guard lock(configLock_); + return config_->clone(); + } + + bool hasNewConfig(const Config& oldConfig); + int contextCountForGpu(uint32_t gpu); + + void handleOnDemandSignal(); + + static void setDaemonConfigLoaderFactory( + std::function()> factory); + + private: + ConfigLoader(); + ~ConfigLoader(); + + const char* configFileName(); + DaemonConfigLoader* daemonConfigLoader(); + + void startThread(); + void updateConfigThread(); + void updateBaseConfig(); + + // Create configuration when receiving SIGUSR2 + void configureFromSignal( + std::chrono::time_point now, + Config& config); + + // Create configuration when receiving request from a daemon + void configureFromDaemon( + std::chrono::time_point now, + Config& config); + + std::string readOnDemandConfigFromDaemon( + std::chrono::time_point now); + + std::mutex configLock_; + std::atomic configFileName_{nullptr}; + std::unique_ptr config_; + std::unique_ptr daemonConfigLoader_; + std::map> handlers_; + + std::chrono::seconds configUpdateIntervalSecs_; + std::chrono::seconds onDemandConfigUpdateIntervalSecs_; + std::unique_ptr updateThread_; + std::condition_variable updateThreadCondVar_; + std::mutex updateThreadMutex_; + std::atomic_bool stopFlag_{false}; + std::atomic_bool onDemandSignal_{false}; + +#if !USE_GOOGLE_LOG + std::unique_ptr> loggerObservers_; + std::mutex loggerObserversMutex_; +#endif // !USE_GOOGLE_LOG +}; + +} // namespace KINETO_NAMESPACE diff --git a/tb_plugins/profiling/libkineto/src/CudaDeviceProperties.cpp b/tb_plugins/profiling/libkineto/src/CudaDeviceProperties.cpp new file mode 100644 index 000000000..1e909d5f9 --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/CudaDeviceProperties.cpp @@ -0,0 +1,130 @@ +/* + * Copyright (c) Kineto Contributors + * All rights reserved. + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "CudaDeviceProperties.h" + +#include +#include + +#include +#include + +#include "Logger.h" + +namespace KINETO_NAMESPACE { + +static const std::vector createDeviceProps() { + std::vector props; + int device_count; + cudaError_t error_id = cudaGetDeviceCount(&device_count); + // Return empty vector if error. + if (error_id != cudaSuccess) { + LOG(ERROR) << "cudaGetDeviceCount failed with code " << error_id; + return {}; + } + VLOG(0) << "Device count is " << device_count; + for (size_t i = 0; i < device_count; ++i) { + cudaDeviceProp prop; + error_id = cudaGetDeviceProperties(&prop, i); + // Return empty vector if any device property fail to get. + if (error_id != cudaSuccess) { + LOG(ERROR) << "cudaGetDeviceProperties failed with " << error_id; + return {}; + } + props.push_back(prop); + LOGGER_OBSERVER_ADD_DEVICE(i); + } + return props; +} + +static const std::vector& deviceProps() { + static const std::vector props = createDeviceProps(); + return props; +} + +static const std::string createDevicePropertiesJson( + size_t id, const cudaDeviceProp& props) { + return fmt::format(R"JSON( + {{ + "id": {}, "name": "{}", "totalGlobalMem": {}, + "computeMajor": {}, "computeMinor": {}, + "maxThreadsPerBlock": {}, "maxThreadsPerMultiprocessor": {}, + "regsPerBlock": {}, "regsPerMultiprocessor": {}, "warpSize": {}, + "sharedMemPerBlock": {}, "sharedMemPerMultiprocessor": {}, + "numSms": {}, "sharedMemPerBlockOptin": {} + }})JSON", + id, props.name, props.totalGlobalMem, + props.major, props.minor, + props.maxThreadsPerBlock, props.maxThreadsPerMultiProcessor, + props.regsPerBlock, props.regsPerMultiprocessor, props.warpSize, + props.sharedMemPerBlock, props.sharedMemPerMultiprocessor, + props.multiProcessorCount, props.sharedMemPerBlockOptin); +} + +static const std::string createDevicePropertiesJson() { + std::vector jsonProps; + const auto& props = deviceProps(); + for (size_t i = 0; i < props.size(); i++) { + jsonProps.push_back(createDevicePropertiesJson(i, props[i])); + } + return fmt::format("{}", fmt::join(jsonProps, ",")); +} + +const std::string& devicePropertiesJson() { + static std::string devicePropsJson = createDevicePropertiesJson(); + return devicePropsJson; +} + +int smCount(uint32_t deviceId) { + const std::vector &props = deviceProps(); + return deviceId >= props.size() ? 0 : + props[deviceId].multiProcessorCount; +} + +float kernelOccupancy( + uint32_t deviceId, + uint16_t registersPerThread, + int32_t staticSharedMemory, + int32_t dynamicSharedMemory, + int32_t blockX, + int32_t blockY, + int32_t blockZ, + float blocksPerSm) { + // Calculate occupancy + float occupancy = -1.0; + const std::vector &props = deviceProps(); + if (deviceId < props.size()) { + cudaOccFuncAttributes occFuncAttr; + occFuncAttr.maxThreadsPerBlock = INT_MAX; + occFuncAttr.numRegs = registersPerThread; + occFuncAttr.sharedSizeBytes = staticSharedMemory; + occFuncAttr.partitionedGCConfig = PARTITIONED_GC_OFF; + occFuncAttr.shmemLimitConfig = FUNC_SHMEM_LIMIT_DEFAULT; + occFuncAttr.maxDynamicSharedSizeBytes = 0; + const cudaOccDeviceState occDeviceState = {}; + int blockSize = blockX * blockY * blockZ; + size_t dynamicSmemSize = dynamicSharedMemory; + cudaOccResult occ_result; + cudaOccDeviceProp prop(props[deviceId]); + cudaOccError status = cudaOccMaxActiveBlocksPerMultiprocessor( + &occ_result, &prop, &occFuncAttr, &occDeviceState, + blockSize, dynamicSmemSize); + if (status == CUDA_OCC_SUCCESS) { + if (occ_result.activeBlocksPerMultiprocessor < blocksPerSm) { + blocksPerSm = occ_result.activeBlocksPerMultiprocessor; + } + occupancy = blocksPerSm * blockSize / + (float) props[deviceId].maxThreadsPerMultiProcessor; + } else { + LOG_EVERY_N(ERROR, 1000) << "Failed to calculate occupancy, status = " + << status; + } + } + return occupancy; +} + +} // namespace KINETO_NAMESPACE diff --git a/tb_plugins/profiling/libkineto/src/CudaDeviceProperties.h b/tb_plugins/profiling/libkineto/src/CudaDeviceProperties.h new file mode 100644 index 000000000..b731fde0c --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/CudaDeviceProperties.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) Kineto Contributors + * All rights reserved. + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include +#include + +namespace KINETO_NAMESPACE { + +int smCount(uint32_t deviceId); + +// Return estimated achieved occupancy for a kernel +float kernelOccupancy( + uint32_t deviceId, + uint16_t registersPerThread, + int32_t staticSharedMemory, + int32_t dynamicSharedMemory, + int32_t blockX, + int32_t blockY, + int32_t blockZ, + float blocks_per_sm); + +// Return compute properties for each device as a json string +const std::string& devicePropertiesJson(); + +} // namespace KINETO_NAMESPACE diff --git a/tb_plugins/profiling/libkineto/src/CuptiActivity.h b/tb_plugins/profiling/libkineto/src/CuptiActivity.h new file mode 100644 index 000000000..09c295040 --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/CuptiActivity.h @@ -0,0 +1,114 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#pragma once + +#include + +#include "ITraceActivity.h" +#include "CuptiActivityPlatform.h" +#include "ThreadUtil.h" +#include "cupti_strings.h" + +namespace libkineto { + class ActivityLogger; +} + +namespace KINETO_NAMESPACE { + +using namespace libkineto; +struct TraceSpan; + +// These classes wrap the various CUPTI activity types +// into subclasses of ITraceActivity so that they can all be accessed +// using the ITraceActivity interface and logged via ActivityLogger. + +// Abstract base class, templated on Cupti activity type +template +struct CuptiActivity : public ITraceActivity { + explicit CuptiActivity(const T* activity, const ITraceActivity* linked) + : activity_(*activity), linked_(linked) {} + int64_t timestamp() const override { + return nsToUs(unixEpochTimestamp(activity_.start)); + } + int64_t duration() const override { + return nsToUs(activity_.end - activity_.start); + } + // TODO(T107507796): Deprecate ITraceActivity + int64_t correlationId() const override {return 0;} + int32_t getThreadId() const override {return 0;} + const ITraceActivity* linkedActivity() const override {return linked_;} + int flowType() const override {return kLinkAsyncCpuGpu;} + int flowId() const override {return correlationId();} + const T& raw() const {return activity_;} + const TraceSpan* traceSpan() const override {return nullptr;} + + protected: + const T& activity_; + const ITraceActivity* linked_{nullptr}; +}; + +// CUpti_ActivityAPI - CUDA runtime activities +struct RuntimeActivity : public CuptiActivity { + explicit RuntimeActivity( + const CUpti_ActivityAPI* activity, + const ITraceActivity* linked, + int32_t threadId) + : CuptiActivity(activity, linked), threadId_(threadId) {} + int64_t correlationId() const override {return activity_.correlationId;} + int64_t deviceId() const override {return processId();} + int64_t resourceId() const override {return threadId_;} + ActivityType type() const override {return ActivityType::CUDA_RUNTIME;} + bool flowStart() const override; + const std::string name() const override {return runtimeCbidName(activity_.cbid);} + void log(ActivityLogger& logger) const override; + const std::string metadataJson() const override; + + private: + const int32_t threadId_; +}; + +// CUpti_ActivityAPI - CUDA runtime activities +struct OverheadActivity : public CuptiActivity { + explicit OverheadActivity( + const CUpti_ActivityOverhead* activity, + const ITraceActivity* linked, + int32_t threadId=0) + : CuptiActivity(activity, linked), threadId_(threadId) {} + + int64_t timestamp() const override { + return nsToUs(unixEpochTimestamp(activity_.start)); + } + int64_t duration() const override { + return nsToUs(activity_.end - activity_.start); + } + // TODO: Update this with PID ordering + int64_t deviceId() const override {return -1;} + int64_t resourceId() const override {return threadId_;} + ActivityType type() const override {return ActivityType::OVERHEAD;} + bool flowStart() const override; + const std::string name() const override {return overheadKindString(activity_.overheadKind);} + void log(ActivityLogger& logger) const override; + const std::string metadataJson() const override; + + private: + const int32_t threadId_; +}; + +// Base class for GPU activities. +// Can also be instantiated directly. +template +struct GpuActivity : public CuptiActivity { + explicit GpuActivity(const T* activity, const ITraceActivity* linked) + : CuptiActivity(activity, linked) {} + int64_t correlationId() const override {return raw().correlationId;} + int64_t deviceId() const override {return raw().deviceId;} + int64_t resourceId() const override {return raw().streamId;} + ActivityType type() const override; + bool flowStart() const override {return false;} + const std::string name() const override; + void log(ActivityLogger& logger) const override; + const std::string metadataJson() const override; + const T& raw() const {return CuptiActivity::raw();} +}; + +} // namespace KINETO_NAMESPACE diff --git a/tb_plugins/profiling/libkineto/src/CuptiActivity.tpp b/tb_plugins/profiling/libkineto/src/CuptiActivity.tpp new file mode 100644 index 000000000..1ff2dafe0 --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/CuptiActivity.tpp @@ -0,0 +1,111 @@ + /* + * Copyright (c) Facebook, Inc. and its affiliates. + * All rights reserved. + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "CuptiActivity.h" + +#include + +#include "Demangle.h" +#include "output_base.h" + +namespace KINETO_NAMESPACE { + +using namespace libkineto; + +template<> +inline const std::string GpuActivity::name() const { + return demangle(raw().name); +} + +template<> +inline ActivityType GpuActivity::type() const { + return ActivityType::CONCURRENT_KERNEL; +} + +static inline std::string memcpyName(uint8_t kind, uint8_t src, uint8_t dst) { + return fmt::format( + "Memcpy {} ({} -> {})", + memcpyKindString((CUpti_ActivityMemcpyKind)kind), + memoryKindString((CUpti_ActivityMemoryKind)src), + memoryKindString((CUpti_ActivityMemoryKind)dst)); +} + +template<> +inline ActivityType GpuActivity::type() const { + return ActivityType::GPU_MEMCPY; +} + +template<> +inline const std::string GpuActivity::name() const { + return memcpyName(raw().copyKind, raw().srcKind, raw().dstKind); +} + +template<> +inline ActivityType GpuActivity::type() const { + return ActivityType::GPU_MEMCPY; +} + +template<> +inline const std::string GpuActivity::name() const { + return memcpyName(raw().copyKind, raw().srcKind, raw().dstKind); +} + +template<> +inline const std::string GpuActivity::name() const { + const char* memory_kind = + memoryKindString((CUpti_ActivityMemoryKind)raw().memoryKind); + return fmt::format("Memset ({})", memory_kind); +} + +template<> +inline ActivityType GpuActivity::type() const { + return ActivityType::GPU_MEMSET; +} + +inline void RuntimeActivity::log(ActivityLogger& logger) const { + logger.handleActivity(*this); +} + +inline void OverheadActivity::log(ActivityLogger& logger) const { + logger.handleActivity(*this); +} + +inline bool OverheadActivity::flowStart() const { + return false; +} + +inline const std::string OverheadActivity::metadataJson() const { + return ""; +} + +template +inline void GpuActivity::log(ActivityLogger& logger) const { + logger.handleGpuActivity(*this); +} + +inline bool RuntimeActivity::flowStart() const { + return activity_.cbid == CUPTI_RUNTIME_TRACE_CBID_cudaLaunchKernel_v7000 || + (activity_.cbid >= CUPTI_RUNTIME_TRACE_CBID_cudaMemcpy_v3020 && + activity_.cbid <= CUPTI_RUNTIME_TRACE_CBID_cudaMemset2DAsync_v3020) || + activity_.cbid == + CUPTI_RUNTIME_TRACE_CBID_cudaLaunchCooperativeKernel_v9000 || + activity_.cbid == + CUPTI_RUNTIME_TRACE_CBID_cudaLaunchCooperativeKernelMultiDevice_v9000; +} + +inline const std::string RuntimeActivity::metadataJson() const { + return fmt::format(R"JSON( + "cbid": {}, "correlation": {})JSON", + activity_.cbid, activity_.correlationId); +} + +template +inline const std::string GpuActivity::metadataJson() const { + return ""; +} + +} // namespace KINETO_NAMESPACE diff --git a/tb_plugins/profiling/libkineto/src/CuptiActivityApi.cpp b/tb_plugins/profiling/libkineto/src/CuptiActivityApi.cpp new file mode 100644 index 000000000..5718bed2f --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/CuptiActivityApi.cpp @@ -0,0 +1,343 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#include "CuptiActivityApi.h" + +#include +#include + +#include "cupti_call.h" +#include "Logger.h" + +using namespace std::chrono; + +namespace KINETO_NAMESPACE { + +// TODO: do we want this to be configurable? +// Set to 2MB to avoid constantly creating buffers (espeically for networks +// that has many small memcpy such as sparseNN) +// Consider putting this on huge pages? +constexpr size_t kBufSize(2 * 1024 * 1024); + +CuptiActivityApi& CuptiActivityApi::singleton() { + static CuptiActivityApi instance; + return instance; +} + +void CuptiActivityApi::pushCorrelationID(int id, CorrelationFlowType type) { +#ifdef HAS_CUPTI + if (!singleton().externalCorrelationEnabled_) { + return; + } + VLOG(2) << "pushCorrelationID(" << id << ")"; + switch(type) { + case Default: + CUPTI_CALL(cuptiActivityPushExternalCorrelationId( + CUPTI_EXTERNAL_CORRELATION_KIND_CUSTOM0, id)); + break; + case User: + CUPTI_CALL(cuptiActivityPushExternalCorrelationId( + CUPTI_EXTERNAL_CORRELATION_KIND_CUSTOM1, id)); + } +#endif +} + +void CuptiActivityApi::popCorrelationID(CorrelationFlowType type) { +#ifdef HAS_CUPTI + if (!singleton().externalCorrelationEnabled_) { + return; + } + switch(type) { + case Default: + CUPTI_CALL(cuptiActivityPopExternalCorrelationId( + CUPTI_EXTERNAL_CORRELATION_KIND_CUSTOM0, nullptr)); + break; + case User: + CUPTI_CALL(cuptiActivityPopExternalCorrelationId( + CUPTI_EXTERNAL_CORRELATION_KIND_CUSTOM1, nullptr)); + } +#endif +} + +static int getSMCount() { +#ifdef HAS_CUPTI + // There may be a simpler way to get the number of SMs.... + // Look for domain_d - this has 80 instances on Volta and + // 56 instances on Pascal, corresponding to the number of SMs + // FIXME: This does not work on Turing and later + uint32_t domainCount{0}; + CUPTI_CALL(cuptiDeviceGetNumEventDomains(0, &domainCount)); + std::vector ids(domainCount); + size_t sz = sizeof(CUpti_EventDomainID) * domainCount; + CUPTI_CALL(cuptiDeviceEnumEventDomains(0, &sz, ids.data())); + for (CUpti_EventDomainID id : ids) { + char name[16]; + name[0] = '\0'; + sz = sizeof(name); + CUPTI_CALL(cuptiEventDomainGetAttribute( + id, CUPTI_EVENT_DOMAIN_ATTR_NAME, &sz, name)); + if (strncmp(name, "domain_d", sz) == 0) { + uint32_t count{0}; + sz = sizeof(count); + CUPTI_CALL(cuptiDeviceGetEventDomainAttribute( + 0, id, CUPTI_EVENT_DOMAIN_ATTR_TOTAL_INSTANCE_COUNT, &sz, &count)); + return count; + } + } +#endif + + return -1; +} + +int CuptiActivityApi::smCount() { + static int sm_count = getSMCount(); + return sm_count; +} + +static bool nextActivityRecord( + uint8_t* buffer, + size_t valid_size, + CUpti_Activity*& record) { +#ifdef HAS_CUPTI + CUptiResult status = CUPTI_CALL_NOWARN( + cuptiActivityGetNextRecord(buffer, valid_size, &record)); + if (status != CUPTI_SUCCESS) { + if (status != CUPTI_ERROR_MAX_LIMIT_REACHED) { + CUPTI_CALL(status); + } + record = nullptr; + } +#endif + return record != nullptr; +} + +void CuptiActivityApi::setMaxBufferSize(int size) { + maxGpuBufferCount_ = 1 + size / kBufSize; +} + +void CuptiActivityApi::forceLoadCupti() { +#ifdef HAS_CUPTI + CUPTI_CALL(cuptiActivityEnable(CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL)); +#endif +} + +#ifdef HAS_CUPTI +void CUPTIAPI CuptiActivityApi::bufferRequestedTrampoline( + uint8_t** buffer, + size_t* size, + size_t* maxNumRecords) { + singleton().bufferRequested(buffer, size, maxNumRecords); +} + +void CuptiActivityApi::bufferRequested( + uint8_t** buffer, size_t* size, size_t* maxNumRecords) { + std::lock_guard guard(mutex_); + if (allocatedGpuTraceBuffers_.size() >= maxGpuBufferCount_) { + stopCollection = true; + LOG(WARNING) << "Exceeded max GPU buffer count (" + << allocatedGpuTraceBuffers_.size() + << " > " << maxGpuBufferCount_ + << ") - terminating tracing"; + } + + auto buf = std::make_unique(kBufSize); + *buffer = buf->data(); + *size = kBufSize; + + allocatedGpuTraceBuffers_[*buffer] = std::move(buf); + + *maxNumRecords = 0; +} +#endif + +std::unique_ptr +CuptiActivityApi::activityBuffers() { + { + std::lock_guard guard(mutex_); + if (allocatedGpuTraceBuffers_.empty()) { + return nullptr; + } + } + +#ifdef HAS_CUPTI + VLOG(1) << "Flushing GPU activity buffers"; + time_point t1; + if (VLOG_IS_ON(1)) { + t1 = system_clock::now(); + } + // Can't hold mutex_ during this call, since bufferCompleted + // will be called by libcupti and mutex_ is acquired there. + CUPTI_CALL(cuptiActivityFlushAll(CUPTI_ACTIVITY_FLAG_FLUSH_FORCED)); + if (VLOG_IS_ON(1)) { + flushOverhead = + duration_cast(system_clock::now() - t1).count(); + } +#endif + std::lock_guard guard(mutex_); + // Transfer ownership of buffers to caller. A new map is created on-demand. + return std::move(readyGpuTraceBuffers_); +} + +#ifdef HAS_CUPTI +int CuptiActivityApi::processActivitiesForBuffer( + uint8_t* buf, + size_t validSize, + std::function handler) { + int count = 0; + if (buf && validSize) { + CUpti_Activity* record{nullptr}; + while ((nextActivityRecord(buf, validSize, record))) { + handler(record); + ++count; + } + } + return count; +} +#endif + +const std::pair CuptiActivityApi::processActivities( + CuptiActivityBufferMap& buffers, + std::function handler) { + std::pair res{0, 0}; +#ifdef HAS_CUPTI + for (auto& pair : buffers) { + // No lock needed - only accessed from this thread + auto& buf = pair.second; + res.first += processActivitiesForBuffer(buf->data(), buf->size(), handler); + res.second += buf->size(); + } +#endif + return res; +} + +void CuptiActivityApi::clearActivities() { + { + std::lock_guard guard(mutex_); + if (allocatedGpuTraceBuffers_.empty()) { + return; + } + } + // Can't hold mutex_ during this call, since bufferCompleted + // will be called by libcupti and mutex_ is acquired there. +#ifdef HAS_CUPTI + CUPTI_CALL(cuptiActivityFlushAll(0)); +#endif + // FIXME: We might want to make sure we reuse + // the same memory during warmup and tracing. + // Also, try to use the amount of memory required + // for active tracing during warmup. + std::lock_guard guard(mutex_); + // Throw away ready buffers as a result of above flush + readyGpuTraceBuffers_ = nullptr; +} + +#ifdef HAS_CUPTI +void CUPTIAPI CuptiActivityApi::bufferCompletedTrampoline( + CUcontext ctx, + uint32_t streamId, + uint8_t* buffer, + size_t /* unused */, + size_t validSize) { + singleton().bufferCompleted(ctx, streamId, buffer, 0, validSize); +} + +void CuptiActivityApi::bufferCompleted( + CUcontext ctx, + uint32_t streamId, + uint8_t* buffer, + size_t /* unused */, + size_t validSize) { + + std::lock_guard guard(mutex_); + auto it = allocatedGpuTraceBuffers_.find(buffer); + if (it == allocatedGpuTraceBuffers_.end()) { + LOG(ERROR) << "bufferCompleted called with unknown buffer: " + << (void*) buffer; + return; + } + + if (!readyGpuTraceBuffers_) { + readyGpuTraceBuffers_ = std::make_unique(); + } + // Set valid size of buffer before moving to ready map + it->second->setSize(validSize); + (*readyGpuTraceBuffers_)[it->first] = std::move(it->second); + allocatedGpuTraceBuffers_.erase(it); + + // report any records dropped from the queue; to avoid unnecessary cupti + // API calls, we make it report only in verbose mode (it doesn't happen + // often in our testing anyways) + if (VLOG_IS_ON(1)) { + size_t dropped = 0; + CUPTI_CALL(cuptiActivityGetNumDroppedRecords(ctx, streamId, &dropped)); + if (dropped != 0) { + LOG(WARNING) << "Dropped " << dropped << " activity records"; + } + } +} +#endif + +void CuptiActivityApi::enableCuptiActivities( + const std::set& selected_activities) { +#ifdef HAS_CUPTI + static bool registered = false; + if (!registered) { + CUPTI_CALL( + cuptiActivityRegisterCallbacks(bufferRequestedTrampoline, bufferCompletedTrampoline)); + } + + externalCorrelationEnabled_ = false; + for (const auto& activity : selected_activities) { + if (activity == ActivityType::GPU_MEMCPY) { + CUPTI_CALL(cuptiActivityEnable(CUPTI_ACTIVITY_KIND_MEMCPY)); + } + if (activity == ActivityType::GPU_MEMSET) { + CUPTI_CALL(cuptiActivityEnable(CUPTI_ACTIVITY_KIND_MEMSET)); + } + if (activity == ActivityType::CONCURRENT_KERNEL) { + CUPTI_CALL(cuptiActivityEnable(CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL)); + } + if (activity == ActivityType::EXTERNAL_CORRELATION) { + CUPTI_CALL(cuptiActivityEnable(CUPTI_ACTIVITY_KIND_EXTERNAL_CORRELATION)); + externalCorrelationEnabled_ = true; + } + if (activity == ActivityType::CUDA_RUNTIME) { + CUPTI_CALL(cuptiActivityEnable(CUPTI_ACTIVITY_KIND_RUNTIME)); + } + if (activity == ActivityType::OVERHEAD) { + CUPTI_CALL(cuptiActivityEnable(CUPTI_ACTIVITY_KIND_OVERHEAD)); + } + } +#endif + + // Explicitly enabled, so reset this flag if set + stopCollection = false; +} + +void CuptiActivityApi::disableCuptiActivities( + const std::set& selected_activities) { +#ifdef HAS_CUPTI + for (const auto& activity : selected_activities) { + if (activity == ActivityType::GPU_MEMCPY) { + CUPTI_CALL(cuptiActivityDisable(CUPTI_ACTIVITY_KIND_MEMCPY)); + } + if (activity == ActivityType::GPU_MEMSET) { + CUPTI_CALL(cuptiActivityDisable(CUPTI_ACTIVITY_KIND_MEMSET)); + } + if (activity == ActivityType::CONCURRENT_KERNEL) { + CUPTI_CALL(cuptiActivityDisable(CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL)); + } + if (activity == ActivityType::EXTERNAL_CORRELATION) { + CUPTI_CALL(cuptiActivityDisable(CUPTI_ACTIVITY_KIND_EXTERNAL_CORRELATION)); + } + if (activity == ActivityType::CUDA_RUNTIME) { + CUPTI_CALL(cuptiActivityDisable(CUPTI_ACTIVITY_KIND_RUNTIME)); + } + if (activity == ActivityType::OVERHEAD) { + CUPTI_CALL(cuptiActivityDisable(CUPTI_ACTIVITY_KIND_OVERHEAD)); + } + } + externalCorrelationEnabled_ = false; +#endif +} + +} // namespace KINETO_NAMESPACE diff --git a/tb_plugins/profiling/libkineto/src/CuptiActivityApi.h b/tb_plugins/profiling/libkineto/src/CuptiActivityApi.h new file mode 100644 index 000000000..92af51eca --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/CuptiActivityApi.h @@ -0,0 +1,100 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#ifdef HAS_CUPTI +#include +#endif + +#include "ActivityType.h" +#include "CuptiActivityBuffer.h" + + +namespace KINETO_NAMESPACE { + +using namespace libkineto; + +#ifndef HAS_CUPTI +using CUpti_Activity = void; +#endif + +class CuptiActivityApi { + public: + enum CorrelationFlowType { + Default, + User + }; + + CuptiActivityApi() = default; + CuptiActivityApi(const CuptiActivityApi&) = delete; + CuptiActivityApi& operator=(const CuptiActivityApi&) = delete; + + virtual ~CuptiActivityApi() {} + + static CuptiActivityApi& singleton(); + + virtual int smCount(); + static void pushCorrelationID(int id, CorrelationFlowType type); + static void popCorrelationID(CorrelationFlowType type); + + void enableCuptiActivities( + const std::set& selected_activities); + void disableCuptiActivities( + const std::set& selected_activities); + void clearActivities(); + + virtual std::unique_ptr activityBuffers(); + + virtual const std::pair processActivities( + CuptiActivityBufferMap&, + std::function handler); + + void setMaxBufferSize(int size); + + std::atomic_bool stopCollection{false}; + int64_t flushOverhead{0}; + + static void forceLoadCupti(); + + private: +#ifdef HAS_CUPTI + int processActivitiesForBuffer( + uint8_t* buf, + size_t validSize, + std::function handler); + static void CUPTIAPI + bufferRequestedTrampoline(uint8_t** buffer, size_t* size, size_t* maxNumRecords); + static void CUPTIAPI bufferCompletedTrampoline( + CUcontext ctx, + uint32_t streamId, + uint8_t* buffer, + size_t /* unused */, + size_t validSize); +#endif // HAS_CUPTI + + int maxGpuBufferCount_{0}; + CuptiActivityBufferMap allocatedGpuTraceBuffers_; + std::unique_ptr readyGpuTraceBuffers_; + std::mutex mutex_; + bool externalCorrelationEnabled_{false}; + + protected: +#ifdef HAS_CUPTI + void bufferRequested(uint8_t** buffer, size_t* size, size_t* maxNumRecords); + void bufferCompleted( + CUcontext ctx, + uint32_t streamId, + uint8_t* buffer, + size_t /* unused */, + size_t validSize); +#endif +}; + +} // namespace KINETO_NAMESPACE diff --git a/tb_plugins/profiling/libkineto/src/CuptiActivityBuffer.h b/tb_plugins/profiling/libkineto/src/CuptiActivityBuffer.h new file mode 100644 index 000000000..1c3fbef62 --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/CuptiActivityBuffer.h @@ -0,0 +1,51 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "ITraceActivity.h" + +namespace KINETO_NAMESPACE { + +class CuptiActivityBuffer { + public: + explicit CuptiActivityBuffer(size_t size) : size_(size) { + buf_.reserve(size); + } + CuptiActivityBuffer() = delete; + CuptiActivityBuffer& operator=(const CuptiActivityBuffer&) = delete; + CuptiActivityBuffer(CuptiActivityBuffer&&) = default; + CuptiActivityBuffer& operator=(CuptiActivityBuffer&&) = default; + + size_t size() const { + return size_; + } + + void setSize(size_t size) { + assert(size <= buf_.capacity()); + size_ = size; + } + + uint8_t* data() { + return buf_.data(); + } + + private: + + std::vector buf_; + size_t size_; + + std::vector> wrappers_; +}; + +using CuptiActivityBufferMap = + std::map>; + +} // namespace KINETO_NAMESPACE diff --git a/tb_plugins/profiling/libkineto/src/CuptiActivityPlatform.cpp b/tb_plugins/profiling/libkineto/src/CuptiActivityPlatform.cpp new file mode 100644 index 000000000..fa2ef2f3a --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/CuptiActivityPlatform.cpp @@ -0,0 +1,31 @@ +#include + +namespace chrono = std::chrono; + +namespace KINETO_NAMESPACE { + +#ifdef _WIN32 +uint64_t epochs_diff() { + // On Windows, steady_clock wraps the QueryPerformanceCounter function. + // https://docs.microsoft.com/en-us/cpp/standard-library/steady-clock-struct?view=msvc-160 + auto steady = + chrono::time_point_cast(chrono::steady_clock::now()); + auto system = + chrono::time_point_cast(chrono::system_clock::now()); + + auto time_since_unix = system.time_since_epoch().count(); + auto time_since_boot = steady.time_since_epoch().count(); + return time_since_unix - time_since_boot; +} + +uint64_t unixEpochTimestamp(uint64_t ts) { + static uint64_t diff = epochs_diff(); + return ts + diff; +} +#else +uint64_t unixEpochTimestamp(uint64_t ts) { + return ts; +} +#endif // _WIN32 + +} // namespace KINETO_NAMESPACE diff --git a/tb_plugins/profiling/libkineto/src/CuptiActivityPlatform.h b/tb_plugins/profiling/libkineto/src/CuptiActivityPlatform.h new file mode 100644 index 000000000..78de8373d --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/CuptiActivityPlatform.h @@ -0,0 +1,12 @@ +#pragma once + +#include + +namespace KINETO_NAMESPACE { + +// cupti's timestamps are platform specific. This function convert the raw +// cupti timestamp to time since unix epoch. So that on different platform, +// correction can work correctly. +uint64_t unixEpochTimestamp(uint64_t ts); + +} // namespace KINETO_NAMESPACE diff --git a/tb_plugins/profiling/libkineto/src/CuptiActivityProfiler.cpp b/tb_plugins/profiling/libkineto/src/CuptiActivityProfiler.cpp new file mode 100644 index 000000000..97c23ef04 --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/CuptiActivityProfiler.cpp @@ -0,0 +1,841 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#include "CuptiActivityProfiler.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef HAS_CUPTI +#include +#endif + +#include "Config.h" +#include "time_since_epoch.h" +#ifdef HAS_CUPTI +#include "CuptiActivity.h" +#include "CuptiActivity.tpp" +#include "CuptiActivityApi.h" +#endif // HAS_CUPTI +#ifdef HAS_ROCTRACER +#include "RoctracerActivityApi.h" +#endif +#include "output_base.h" + +#include "Logger.h" +#include "ThreadUtil.h" + +using namespace std::chrono; +using namespace libkineto; +using std::string; + +namespace KINETO_NAMESPACE { + +void CuptiActivityProfiler::transferCpuTrace( + std::unique_ptr cpuTrace) { + std::lock_guard guard(mutex_); + const string& trace_name = cpuTrace->span.name; + if (currentRunloopState_ != RunloopState::CollectTrace && + currentRunloopState_ != RunloopState::ProcessTrace) { + VLOG(0) << "Trace collection not in progress - discarding span " + << trace_name; + return; + } + + cpuTrace->span.iteration = iterationCountMap_[trace_name]++; + + VLOG(0) << "Received iteration " << cpuTrace->span.iteration << " of span " + << trace_name << " (" << cpuTrace->activities.size() << " activities / " + << cpuTrace->gpuOpCount << " gpu activities)"; + traceBuffers_->cpu.push_back(std::move(cpuTrace)); +} + +#ifdef HAS_ROCTRACER +CuptiActivityProfiler::CuptiActivityProfiler(RoctracerActivityApi& cupti, bool cpuOnly) +#else +CuptiActivityProfiler::CuptiActivityProfiler(CuptiActivityApi& cupti, bool cpuOnly) +#endif + : cupti_(cupti), + flushOverhead_{0, 0}, + setupOverhead_{0, 0}, + cpuOnly_{cpuOnly}, + currentRunloopState_{RunloopState::WaitForRequest}, + stopCollection_{false} {} + +void CuptiActivityProfiler::processTraceInternal(ActivityLogger& logger) { + LOG(INFO) << "Processing " << traceBuffers_->cpu.size() + << " CPU buffers"; + VLOG(0) << "Profile time range: " << captureWindowStartTime_ << " - " + << captureWindowEndTime_; + logger.handleTraceStart(metadata_); + for (auto& cpu_trace : traceBuffers_->cpu) { + string trace_name = cpu_trace->span.name; + VLOG(0) << "Processing CPU buffer for " << trace_name << " (" + << cpu_trace->span.iteration << ") - " + << cpu_trace->activities.size() << " records"; + VLOG(0) << "Span time range: " << cpu_trace->span.startTime << " - " + << cpu_trace->span.endTime; + processCpuTrace(*cpu_trace, logger); + LOGGER_OBSERVER_ADD_EVENT_COUNT(cpu_trace->activities.size()); + } + +#ifdef HAS_CUPTI + if (!cpuOnly_) { + VLOG(0) << "Retrieving GPU activity buffers"; + traceBuffers_->gpu = cupti_.activityBuffers(); + if (VLOG_IS_ON(1)) { + addOverheadSample(flushOverhead_, cupti_.flushOverhead); + } + if (traceBuffers_->gpu) { + const auto count_and_size = cupti_.processActivities( + *traceBuffers_->gpu, + std::bind(&CuptiActivityProfiler::handleCuptiActivity, this, std::placeholders::_1, &logger)); + LOG(INFO) << "Processed " << count_and_size.first + << " GPU records (" << count_and_size.second << " bytes)"; + LOGGER_OBSERVER_ADD_EVENT_COUNT(count_and_size.first); + } + } +#endif // HAS_CUPTI +#ifdef HAS_ROCTRACER + if (!cpuOnly_) { + VLOG(0) << "Retrieving GPU activity buffers"; + const int count = cupti_.processActivities(logger); + LOG(INFO) << "Processed " << count + << " GPU records"; + LOGGER_OBSERVER_ADD_EVENT_COUNT(count); + } +#endif // HAS_ROCTRACER + + for (const auto& session : sessions_){ + LOG(INFO) << "Processing child profiler trace"; + session->processTrace(logger); + } + + finalizeTrace(*config_, logger); +} + +CuptiActivityProfiler::CpuGpuSpanPair& CuptiActivityProfiler::recordTraceSpan( + TraceSpan& span, int gpuOpCount) { + TraceSpan gpu_span(gpuOpCount, span.iteration, span.name, "GPU: "); + auto& iterations = traceSpans_[span.name]; + iterations.push_back({span, gpu_span}); + return iterations.back(); +} + +void CuptiActivityProfiler::processCpuTrace( + libkineto::CpuTraceBuffer& cpuTrace, + ActivityLogger& logger) { + if (cpuTrace.activities.size() == 0) { + LOG(WARNING) << "CPU trace is empty!"; + return; + } + + CpuGpuSpanPair& span_pair = recordTraceSpan(cpuTrace.span, cpuTrace.gpuOpCount); + TraceSpan& cpu_span = span_pair.first; + for (auto const& act : cpuTrace.activities) { + VLOG(2) << act.correlationId() << ": OP " << act.activityName; + if (config_->selectedActivityTypes().count(act.type())) { + act.log(logger); + } + clientActivityTraceMap_[act.correlationId()] = &span_pair; + activityMap_[act.correlationId()] = &act; + + recordThreadInfo(act.resourceId(), act.getThreadId(), act.deviceId()); + } + logger.handleTraceSpan(cpu_span); +} + +#ifdef HAS_CUPTI +inline void CuptiActivityProfiler::handleCorrelationActivity( + const CUpti_ActivityExternalCorrelation* correlation) { + if (correlation->externalKind == CUPTI_EXTERNAL_CORRELATION_KIND_CUSTOM0) { + cpuCorrelationMap_[correlation->correlationId] = correlation->externalId; + } else if (correlation->externalKind == CUPTI_EXTERNAL_CORRELATION_KIND_CUSTOM1){ + userCorrelationMap_[correlation->correlationId] = correlation->externalId; + } else { + LOG(ERROR) << "Invalid CUpti_ActivityExternalCorrelation sent to handleCuptiActivity"; + } +} +#endif // HAS_CUPTI + +static GenericTraceActivity createUserGpuSpan( + const libkineto::ITraceActivity& cpuTraceActivity, + const libkineto::ITraceActivity& gpuTraceActivity) { + GenericTraceActivity res( + *cpuTraceActivity.traceSpan(), + ActivityType::GPU_USER_ANNOTATION, + cpuTraceActivity.name()); + res.startTime = gpuTraceActivity.timestamp(); + res.device = gpuTraceActivity.deviceId(); + res.resource = gpuTraceActivity.resourceId(); + res.endTime = + gpuTraceActivity.timestamp() + gpuTraceActivity.duration(); + res.id = cpuTraceActivity.correlationId(); + return res; +} + +void CuptiActivityProfiler::GpuUserEventMap::insertOrExtendEvent( + const ITraceActivity& userActivity, + const ITraceActivity& gpuActivity) { + StreamKey key(gpuActivity.deviceId(), gpuActivity.resourceId()); + CorrelationSpanMap& correlationSpanMap = streamSpanMap_[key]; + auto it = correlationSpanMap.find(userActivity.correlationId()); + if (it == correlationSpanMap.end()) { + auto it_success = correlationSpanMap.insert({ + userActivity.correlationId(), createUserGpuSpan(userActivity, gpuActivity) + }); + it = it_success.first; + } + GenericTraceActivity& span = it->second; + if (gpuActivity.timestamp() < span.startTime || span.startTime == 0) { + span.startTime = gpuActivity.timestamp(); + } + int64_t gpu_activity_end = gpuActivity.timestamp() + gpuActivity.duration(); + if (gpu_activity_end > span.endTime) { + span.endTime = gpu_activity_end; + } +} + +const CuptiActivityProfiler::CpuGpuSpanPair& CuptiActivityProfiler::defaultTraceSpan() { + static TraceSpan span(0, 0, "Unknown", ""); + static CpuGpuSpanPair span_pair(span, span); + return span_pair; +} + +void CuptiActivityProfiler::GpuUserEventMap::logEvents(ActivityLogger *logger) { + for (auto const& streamMapPair : streamSpanMap_) { + for (auto const& correlationSpanPair : streamMapPair.second) { + correlationSpanPair.second.log(*logger); + } + } +} + +#ifdef HAS_CUPTI +inline bool CuptiActivityProfiler::outOfRange(const ITraceActivity& act) { + bool out_of_range = act.timestamp() < captureWindowStartTime_ || + (act.timestamp() + act.duration()) > captureWindowEndTime_; + if (out_of_range) { + VLOG(2) << "TraceActivity outside of profiling window: " << act.name() + << " (" << act.timestamp() << " < " << captureWindowStartTime_ << " or " + << (act.timestamp() + act.duration()) << " > " << captureWindowEndTime_; + } + return out_of_range; +} + +inline static bool isBlockListedRuntimeCbid(CUpti_CallbackId cbid) { + // Some CUDA calls that are very frequent and also not very interesting. + // Filter these out to reduce trace size. + if (cbid == CUPTI_RUNTIME_TRACE_CBID_cudaGetDevice_v3020 || + cbid == CUPTI_RUNTIME_TRACE_CBID_cudaSetDevice_v3020 || + cbid == CUPTI_RUNTIME_TRACE_CBID_cudaGetLastError_v3020 || + // Don't care about cudaEvents + cbid == CUPTI_RUNTIME_TRACE_CBID_cudaEventCreate_v3020 || + cbid == CUPTI_RUNTIME_TRACE_CBID_cudaEventCreateWithFlags_v3020 || + cbid == CUPTI_RUNTIME_TRACE_CBID_cudaEventRecord_v3020 || + cbid == CUPTI_RUNTIME_TRACE_CBID_cudaEventDestroy_v3020 || + cbid == CUPTI_RUNTIME_TRACE_CBID_cudaEventSynchronize_v3020) { + return true; + } + + return false; +} + +void CuptiActivityProfiler::handleRuntimeActivity( + const CUpti_ActivityAPI* activity, + ActivityLogger* logger) { + if (isBlockListedRuntimeCbid(activity->cbid)) { + return; + } + VLOG(2) << activity->correlationId + << ": CUPTI_ACTIVITY_KIND_RUNTIME, cbid=" << activity->cbid + << " tid=" << activity->threadId; + int32_t tid = activity->threadId; + const auto& it = resourceInfo_.find({processId(), tid}); + if (it != resourceInfo_.end()) { + tid = it->second.id; + } + const ITraceActivity* linked = linkedActivity( + activity->correlationId, cpuCorrelationMap_); + const auto& runtime_activity = + traceBuffers_->addActivityWrapper(RuntimeActivity(activity, linked, tid)); + checkTimestampOrder(&runtime_activity); + if (outOfRange(runtime_activity)) { + return; + } + runtime_activity.log(*logger); +} + +void CuptiActivityProfiler::handleOverheadActivity( + const CUpti_ActivityOverhead* activity, + ActivityLogger* logger) { + VLOG(2) << ": CUPTI_ACTIVITY_KIND_OVERHEAD" << " overheadKind=" << activity->overheadKind; + + const auto& overhead_activity = + traceBuffers_->addActivityWrapper(OverheadActivity(activity, nullptr)); + overhead_activity.log(*logger); +} + + +inline void CuptiActivityProfiler::updateGpuNetSpan( + const ITraceActivity& gpuOp) { + if (!gpuOp.linkedActivity()) { + VLOG(0) << "Missing linked activity"; + return; + } + const auto& it = clientActivityTraceMap_.find( + gpuOp.linkedActivity()->correlationId()); + if (it == clientActivityTraceMap_.end()) { + // No correlation id mapping? + return; + } + TraceSpan& gpu_span = it->second->second; + if (gpuOp.timestamp() < gpu_span.startTime || gpu_span.startTime == 0) { + gpu_span.startTime = gpuOp.timestamp(); + } + if ((gpuOp.timestamp() + gpuOp.duration()) > gpu_span.endTime) { + gpu_span.endTime = gpuOp.timestamp() + gpuOp.duration(); + } +} + +// I've observed occasional broken timestamps attached to GPU events... +void CuptiActivityProfiler::checkTimestampOrder(const ITraceActivity* act1) { + // Correlated GPU runtime activity cannot + // have timestamp greater than the GPU activity's + const auto& it = correlatedCudaActivities_.find(act1->correlationId()); + if (it == correlatedCudaActivities_.end()) { + correlatedCudaActivities_.insert({act1->correlationId(), act1}); + return; + } + + // Activities may be appear in the buffers out of order. + // If we have a runtime activity in the map, it should mean that we + // have a GPU activity passed in, and vice versa. + const ITraceActivity* act2 = it->second; + if (act2->type() == ActivityType::CUDA_RUNTIME) { + // Buffer is out-of-order. + // Swap so that runtime activity is first for the comparison below. + std::swap(act1, act2); + } + if (act1->timestamp() > act2->timestamp()) { + LOG(WARNING) << "GPU op timestamp (" << act2->timestamp() + << ") < runtime timestamp (" << act1->timestamp() << ") by " + << act1->timestamp() - act2->timestamp() << "us"; + LOG(WARNING) << "Name: " << act2->name() + << " Device: " << act2->deviceId() + << " Stream: " << act2->resourceId(); + } +} + +inline void CuptiActivityProfiler::handleGpuActivity( + const ITraceActivity& act, + ActivityLogger* logger) { + if (outOfRange(act)) { + return; + } + checkTimestampOrder(&act); + VLOG(2) << act.correlationId() << ": " + << act.name(); + recordStream(act.deviceId(), act.resourceId(), ""); + act.log(*logger); + updateGpuNetSpan(act); + if (config_->selectedActivityTypes().count(ActivityType::GPU_USER_ANNOTATION)) { + const auto& it = userCorrelationMap_.find(act.correlationId()); + if (it != userCorrelationMap_.end()) { + const auto& it2 = activityMap_.find(it->second); + if (it2 != activityMap_.end()) { + recordStream(act.deviceId(), act.resourceId(), "context"); + gpuUserEventMap_.insertOrExtendEvent(*it2->second, act); + } + } + } +} + +const ITraceActivity* CuptiActivityProfiler::linkedActivity( + int32_t correlationId, + const std::unordered_map& correlationMap) { + const auto& it = correlationMap.find(correlationId); + if (it != correlationMap.end()) { + const auto& it2 = activityMap_.find(it->second); + if (it2 != activityMap_.end()) { + return it2->second; + } + } + return nullptr; +} + +template +inline void CuptiActivityProfiler::handleGpuActivity( + const T* act, ActivityLogger* logger) { + const ITraceActivity* linked = linkedActivity( + act->correlationId, cpuCorrelationMap_); + const auto& gpu_activity = + traceBuffers_->addActivityWrapper(GpuActivity(act, linked)); + handleGpuActivity(gpu_activity, logger); +} + +void CuptiActivityProfiler::handleCuptiActivity(const CUpti_Activity* record, ActivityLogger* logger) { + switch (record->kind) { + case CUPTI_ACTIVITY_KIND_EXTERNAL_CORRELATION: + handleCorrelationActivity( + reinterpret_cast( + record)); + break; + case CUPTI_ACTIVITY_KIND_RUNTIME: + handleRuntimeActivity( + reinterpret_cast(record), logger); + break; + case CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL: + handleGpuActivity( + reinterpret_cast(record), logger); + break; + case CUPTI_ACTIVITY_KIND_MEMCPY: + handleGpuActivity( + reinterpret_cast(record), logger); + break; + case CUPTI_ACTIVITY_KIND_MEMCPY2: + handleGpuActivity( + reinterpret_cast(record), logger); + break; + case CUPTI_ACTIVITY_KIND_MEMSET: + handleGpuActivity( + reinterpret_cast(record), logger); + break; + case CUPTI_ACTIVITY_KIND_OVERHEAD: + handleOverheadActivity (reinterpret_cast(record), logger); + break; + default: + LOG(WARNING) << "Unexpected activity type: " << record->kind; + break; + } +} +#endif // HAS_CUPTI + +void CuptiActivityProfiler::configureChildProfilers() { + // If child profilers are enabled create profiler sessions + for (auto& profiler: profilers_) { + int64_t start_time_ms = duration_cast( + profileStartTime_.time_since_epoch()).count(); + LOG(INFO) << "Running child profiler " << profiler->name() << " for " + << config_->activitiesDuration().count() << " ms"; + auto session = profiler->configure( + start_time_ms, + config_->activitiesDuration().count(), + config_->selectedActivityTypes(), + *config_ + ); + if (session) { + sessions_.push_back(std::move(session)); + } + } +} + +void CuptiActivityProfiler::configure( + const Config& config, + const time_point& now) { + std::lock_guard guard(mutex_); + if (isActive()) { + LOG(ERROR) << "CuptiActivityProfiler already busy, terminating"; + return; + } + + config_ = config.clone(); + + if (config_->activitiesDuration().count() == 0) { + // Use default if not specified + config_->setActivitiesDuration( + config_->activitiesDurationDefault()); + } + + // Ensure we're starting in a clean state + resetTraceData(); + +#if !USE_GOOGLE_LOG + // Add a LoggerObserverCollector to collect all logs during the trace. + loggerCollectorMetadata_ = std::make_unique(); + Logger::addLoggerObserver(loggerCollectorMetadata_.get()); +#endif // !USE_GOOGLE_LOG + + profileStartTime_ = config_->requestTimestamp(); + + if (config_->hasProfileStartIteration()) { + profileStartIter_ = config_->profileStartIteration(); + profileEndIter_ = profileStartIter_ + config_->activitiesRunIterations(); + } else { + + profileStartIter_ = -1; + profileEndIter_ = (std::numeric_limits::max)(); + + if (profileStartTime_ < now) { + LOG(ERROR) << "Not starting tracing - start timestamp is in the past. Time difference (ms): " << duration_cast(now - profileStartTime_).count(); + return; + } else if ((profileStartTime_ - now) < config_->activitiesWarmupDuration()) { + LOG(ERROR) << "Not starting tracing - insufficient time for warmup. Time to warmup (ms): " << duration_cast(profileStartTime_ - now).count() ; + return; + } + } + + if (LOG_IS_ON(INFO)) { + config_->printActivityProfilerConfig(LIBKINETO_DBG_STREAM); + } + if (!cpuOnly_ && !libkineto::api().client()) { + if (profileStartIter_ < 0) { + LOG(INFO) << "GPU-only tracing for " + << config_->activitiesDuration().count() << "ms"; + } else { + LOG(INFO) << "GPU-only tracing for " + << config_->activitiesRunIterations() << " iterations"; + } + } + + // Set useful metadata into the logger. + LOGGER_OBSERVER_SET_TRACE_DURATION_MS(config_->activitiesDuration().count()); + if (!config_->requestTraceID().empty()) { + LOGGER_OBSERVER_SET_TRACE_ID(config_->requestTraceID()); + } + if (!config_->requestGroupTraceID().empty()) { + LOGGER_OBSERVER_SET_GROUP_TRACE_ID(config_->requestGroupTraceID()); + } + LOGGER_OBSERVER_ADD_DESTINATION(config_->activitiesLogUrl()); + +#if defined(HAS_CUPTI) || defined(HAS_ROCTRACER) + if (!cpuOnly_) { + // Enabling CUPTI activity tracing incurs a larger perf hit at first, + // presumably because structures are allocated and initialized, callbacks + // are activated etc. After a while the overhead decreases and stabilizes. + // It's therefore useful to perform some warmup before starting recording. + LOG(INFO) << "Enabling GPU tracing"; + cupti_.setMaxBufferSize(config_->activitiesMaxGpuBufferSize()); + + time_point timestamp; + if (VLOG_IS_ON(1)) { + timestamp = system_clock::now(); + } +#ifdef HAS_CUPTI + cupti_.enableCuptiActivities(config_->selectedActivityTypes()); +#else + cupti_.enableActivities(config_->selectedActivityTypes()); +#endif + if (VLOG_IS_ON(1)) { + auto t2 = system_clock::now(); + addOverheadSample( + setupOverhead_, duration_cast(t2 - timestamp).count()); + } + } +#endif // HAS_CUPTI || HAS_ROCTRACER + + if (profilers_.size() > 0) { + configureChildProfilers(); + } + + if (libkineto::api().client()) { + libkineto::api().client()->warmup(config_->isOpInputsCollectionEnabled()); + } + if (profileStartIter_ >= 0) { + LOG(INFO) << "Tracing starting on iteration = " << profileStartIter_; + } else { + LOG(INFO) << "Tracing starting in " + << duration_cast(profileStartTime_ - now).count() << "s"; + } + + traceBuffers_ = std::make_unique(); + captureWindowStartTime_ = captureWindowEndTime_ = 0; + currentRunloopState_ = RunloopState::Warmup; +} + +void CuptiActivityProfiler::startTraceInternal(const time_point& now) { + captureWindowStartTime_ = libkineto::timeSinceEpoch(now); + VLOG(0) << "Warmup -> CollectTrace"; + for (auto& session: sessions_){ + LOG(INFO) << "Starting child profiler session"; + session->start(); + } + currentRunloopState_ = RunloopState::CollectTrace; +} + +void CuptiActivityProfiler::stopTraceInternal(const time_point& now) { + if (captureWindowEndTime_ == 0) { + captureWindowEndTime_ = libkineto::timeSinceEpoch(now); + } +#if defined(HAS_CUPTI) || defined(HAS_ROCTRACER) + if (!cpuOnly_) { + time_point timestamp; + if (VLOG_IS_ON(1)) { + timestamp = system_clock::now(); + } +#ifdef HAS_CUPTI + cupti_.disableCuptiActivities(config_->selectedActivityTypes()); +#else + cupti_.disableActivities(config_->selectedActivityTypes()); +#endif + if (VLOG_IS_ON(1)) { + auto t2 = system_clock::now(); + addOverheadSample( + setupOverhead_, duration_cast(t2 - timestamp).count()); + } + } +#endif // HAS_CUPTI || HAS_ROCTRACER + + if (currentRunloopState_ == RunloopState::CollectTrace) { + VLOG(0) << "CollectTrace -> ProcessTrace"; + } else { + LOG(WARNING) << "Called stopTrace with state == " << + static_cast::type>( + currentRunloopState_.load()); + } + for (auto& session: sessions_){ + LOG(INFO) << "Stopping child profiler session"; + session->stop(); + } + currentRunloopState_ = RunloopState::ProcessTrace; +} + +void CuptiActivityProfiler::resetInternal() { + resetTraceData(); + currentRunloopState_ = RunloopState::WaitForRequest; +} + +bool CuptiActivityProfiler::isWarmupDone( + const time_point& now, + int64_t currentIter) const { + // is it a time based config + if (profileStartIter_ < 0) { + // qualify that this check is not being called from application step() API + // this avoids races between the step() API and periodically invoked + // profiler run loop step() method + return (currentIter < 0) && (now >= profileStartTime_); + } + // this is an iteration based config + if (currentIter < 0) { + return false; + } + return currentIter >= profileStartIter_; +} + +bool CuptiActivityProfiler::isCollectionDone( + const time_point& now, + int64_t currentIter) const { + // is it a time based config + if (profileStartIter_ < 0) { + // qualify that this check is not being called from application step() API + return (currentIter < 0) && (now >= profileEndTime_); + } + // this is an iteration based config + if (currentIter < 0) { + return false; + } + return currentIter >= profileEndIter_; +} + +const time_point CuptiActivityProfiler::performRunLoopStep( + const time_point& now, + const time_point& nextWakeupTime, + int64_t currentIter) { + auto new_wakeup_time = nextWakeupTime; + bool warmup_done = false, collection_done = false; + + VLOG_IF(1, currentIter >= 0) << "Run loop on application step(), iteration = " + << currentIter; + + switch (currentRunloopState_) { + case RunloopState::WaitForRequest: + VLOG(1) << "State: WaitForRequest"; + // Nothing to do + break; + + case RunloopState::Warmup: + VLOG(1) << "State: Warmup"; + warmup_done = isWarmupDone(now, currentIter); +#if defined(HAS_CUPTI) || defined(HAS_ROCTRACER) + // Flushing can take a while so avoid doing it close to the start time + if (!cpuOnly_ && currentIter < 0 && + (profileStartIter_ >= 0 || nextWakeupTime < profileStartTime_)) { + cupti_.clearActivities(); + } + + if (cupti_.stopCollection) { + // Go to process trace to clear any outstanding buffers etc + LOG(WARNING) << "Trace terminated during warmup"; + std::lock_guard guard(mutex_); + stopTraceInternal(now); + resetInternal(); + VLOG(0) << "Warmup -> WaitForRequest"; + break; + } +#endif // HAS_CUPTI || HAS_ROCTRACER + + if (warmup_done) { + UST_LOGGER_MARK_COMPLETED(kWarmUpStage); + if (profileStartIter_ < 0 && + (now > profileStartTime_ + milliseconds(10))) { + LOG(WARNING) + << "Tracing started " + << duration_cast(now - profileStartTime_).count() + << "ms late!"; + } else { + LOG(INFO) << "Tracing started"; + } + startTrace(now); + if (libkineto::api().client()) { + libkineto::api().client()->start(); + } + if (nextWakeupTime > profileEndTime_) { + new_wakeup_time = profileEndTime_; + } + } else if (nextWakeupTime > profileStartTime_) { + new_wakeup_time = profileStartTime_; + } + + break; + + case RunloopState::CollectTrace: + VLOG(1) << "State: CollectTrace"; + // captureWindowStartTime_ can be set by external threads, + // so recompute end time. + // FIXME: Is this a good idea for synced start? + if (profileStartIter_ < 0) { + std::lock_guard guard(mutex_); + profileEndTime_ = time_point( + microseconds(captureWindowStartTime_)) + + config_->activitiesDuration(); + } + + collection_done = isCollectionDone(now, currentIter); + + // TODO revisit stopCollection_ is not used right now + if (collection_done || stopCollection_.exchange(false) +#if defined(HAS_CUPTI) || defined(HAS_ROCTRACER) + || cupti_.stopCollection +#endif // HAS_CUPTI || HAS_ROCTRACER + ){ + // Update runloop state first to prevent further updates to shared state + LOG(INFO) << "Tracing complete."; + if (currentIter > 0) { + LOG(INFO) << "This state change was invoked by application's step() call"; + } + // FIXME: Need to communicate reason for stopping on errors + if (libkineto::api().client()) { + libkineto::api().client()->stop(); + } + std::lock_guard guard(mutex_); + stopTraceInternal(now); + VLOG_IF(0, collection_done) << "Reached profile end time"; + + UST_LOGGER_MARK_COMPLETED(kCollectionStage); + } else if (profileStartIter_ >= 0) { + // nothing to do here + } else if (now < profileEndTime_ && profileEndTime_ < nextWakeupTime) { + new_wakeup_time = profileEndTime_; + } + + break; + + case RunloopState::ProcessTrace: + VLOG(1) << "State: ProcessTrace"; + // skip this state transition if it called from the step() api + // of the profiler. + // else it could lead to a race between the profiler thread and an + // application thread calling step() + if (currentIter >= 0) { + return new_wakeup_time; + } + // FIXME: Probably want to allow interruption here + // for quickly handling trace request via synchronous API + std::lock_guard guard(mutex_); + processTraceInternal(*logger_); + UST_LOGGER_MARK_COMPLETED(kPostProcessingStage); + resetInternal(); + VLOG(0) << "ProcessTrace -> WaitForRequest"; + break; + } + + return new_wakeup_time; +} + +void CuptiActivityProfiler::finalizeTrace(const Config& config, ActivityLogger& logger) { + LOG(INFO) << "Recorded nets:"; + { + for (const auto& it : iterationCountMap_) { + LOG(INFO) << it.first << ": " << it.second << " iterations"; + } + iterationCountMap_.clear(); + } + + // Process names + int32_t pid = processId(); + string process_name = processName(pid); + if (!process_name.empty()) { + logger.handleDeviceInfo( + {pid, process_name, "CPU"}, captureWindowStartTime_); + if (!cpuOnly_) { + // GPU events use device id as pid (0-7). + constexpr int kMaxGpuCount = 8; + for (int gpu = 0; gpu < kMaxGpuCount; gpu++) { + logger.handleDeviceInfo( + {gpu, process_name, fmt::format("GPU {}", gpu)}, + captureWindowStartTime_); + } + } + } + + // Thread & stream info + for (auto pair : resourceInfo_) { + const auto& resource = pair.second; + logger.handleResourceInfo(resource, captureWindowStartTime_); + } + + for (const auto& iterations : traceSpans_) { + for (const auto& span_pair : iterations.second) { + const TraceSpan& gpu_span = span_pair.second; + if (gpu_span.opCount > 0) { + logger.handleTraceSpan(gpu_span); + } + } + } + + // Overhead info + overheadInfo_.push_back(ActivityLogger::OverheadInfo("CUPTI Overhead")); + for(const auto& info : overheadInfo_) { + logger.handleOverheadInfo(info, captureWindowStartTime_); + } + + gpuUserEventMap_.logEvents(&logger); + +#if !USE_GOOGLE_LOG + // Save logs from LoggerCollector objects into Trace metadata. + auto LoggerMD = loggerCollectorMetadata_->extractCollectorMetadata(); + std::unordered_map> LoggerMDString; + for (auto& md : LoggerMD) { + LoggerMDString[toString(md.first)] = md.second; + } +#endif // !USE_GOOGLE_LOG + + logger.finalizeTrace(config, std::move(traceBuffers_), captureWindowEndTime_, LoggerMDString); +} + +void CuptiActivityProfiler::resetTraceData() { +#if defined(HAS_CUPTI) || defined(HAS_ROCTRACER) + if (!cpuOnly_) { + cupti_.clearActivities(); + } +#endif // HAS_CUPTI || HAS_ROCTRACER + activityMap_.clear(); + cpuCorrelationMap_.clear(); + correlatedCudaActivities_.clear(); + gpuUserEventMap_.clear(); + traceSpans_.clear(); + clientActivityTraceMap_.clear(); + traceBuffers_ = nullptr; + metadata_.clear(); + sessions_.clear(); +#if !USE_GOOGLE_LOG + Logger::removeLoggerObserver(loggerCollectorMetadata_.get()); +#endif // !USE_GOOGLE_LOG +} + + +} // namespace KINETO_NAMESPACE diff --git a/tb_plugins/profiling/libkineto/src/CuptiActivityProfiler.h b/tb_plugins/profiling/libkineto/src/CuptiActivityProfiler.h new file mode 100644 index 000000000..208833a4d --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/CuptiActivityProfiler.h @@ -0,0 +1,364 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// TODO(T90238193) +// @lint-ignore-every CLANGTIDY facebook-hte-RelativeInclude +#include "ThreadUtil.h" +#include "TraceSpan.h" +#include "libkineto.h" +#include "output_base.h" +#include "GenericTraceActivity.h" +#include "IActivityProfiler.h" +#include "LoggerCollector.h" + +namespace KINETO_NAMESPACE { + +class Config; +class CuptiActivityApi; +class RoctracerActivityApi; + +class CuptiActivityProfiler { + public: + CuptiActivityProfiler(CuptiActivityApi& cupti, bool cpuOnly); + CuptiActivityProfiler(RoctracerActivityApi& rai, bool cpuOnly); + CuptiActivityProfiler(const CuptiActivityProfiler&) = delete; + CuptiActivityProfiler& operator=(const CuptiActivityProfiler&) = delete; + + bool isActive() const { + return currentRunloopState_ != RunloopState::WaitForRequest; + } + + // Invoke at a regular interval to perform profiling activities. + // When not active, an interval of 1-5 seconds is probably fine, + // depending on required warm-up time and delayed start time. + // When active, it's a good idea to invoke more frequently to stay below + // memory usage limit (ACTIVITIES_MAX_GPU_BUFFER_SIZE_MB) during warmup. + const std::chrono::time_point performRunLoopStep( + const std::chrono::time_point& now, + const std::chrono::time_point& nextWakeupTime, + int64_t currentIter = -1); + + // Used for async requests + void setLogger(ActivityLogger* logger) { + logger_ = logger; + } + + // Synchronous control API + void startTrace( + const std::chrono::time_point& now) { + std::lock_guard guard(mutex_); + startTraceInternal(now); + } + + void stopTrace(const std::chrono::time_point& now) { + std::lock_guard guard(mutex_); + stopTraceInternal(now); + } + + // Process CPU and GPU traces + void processTrace(ActivityLogger& logger) { + std::lock_guard guard(mutex_); + processTraceInternal(logger); + } + + void reset() { + std::lock_guard guard(mutex_); + resetInternal(); + } + + // Set up profiler as specified in config. + void configure( + const Config& config, + const std::chrono::time_point& now); + + // Registered with client API to pass CPU trace events over + void transferCpuTrace( + std::unique_ptr cpuTrace); + + Config& config() { + return *config_; + } + + inline void recordThreadInfo() { + int32_t sysTid = systemThreadId(); + // Note we're using the lower 32 bits of the (opaque) pthread id + // as key, because that's what CUPTI records. + int32_t tid = threadId(); + int32_t pid = processId(); + std::lock_guard guard(mutex_); + recordThreadInfo(sysTid, tid, pid); + } + + // T107508020: We can deprecate the recordThreadInfo(void) once we optimized profiler_kineto + void recordThreadInfo(int32_t sysTid, int32_t tid, int32_t pid) { + if (resourceInfo_.find({pid, tid}) == resourceInfo_.end()) { + resourceInfo_.emplace( + std::make_pair(pid, tid), + ActivityLogger::ResourceInfo( + pid, + sysTid, + sysTid, // sortindex + fmt::format("thread {} ({})", sysTid, getThreadName()))); + } + } + + void addMetadata(const std::string& key, const std::string& value) { + std::lock_guard guard(mutex_); + metadata_[key] = value; + } + + void addChildActivityProfiler( + std::unique_ptr profiler) { + std::lock_guard guard(mutex_); + profilers_.push_back(std::move(profiler)); + } + + protected: + + using CpuGpuSpanPair = std::pair; + static const CpuGpuSpanPair& defaultTraceSpan(); + + private: + + // Map of gpu activities to user defined events + class GpuUserEventMap { + public: + // Insert a user defined event which maps to the gpu trace activity. + // If the user defined event mapping already exists this will update the + // gpu side span to include the span of gpuTraceActivity. + void insertOrExtendEvent(const ITraceActivity& cpuTraceActivity, + const ITraceActivity& gpuTraceActivity); + // Log out the events to the logger + void logEvents(ActivityLogger *logger); + + void clear() { + streamSpanMap_.clear(); + } + + private: + // device id and stream name + using StreamKey = std::pair; + + // map of correlation id to TraceSpan + using CorrelationSpanMap = + std::unordered_map; + std::map streamSpanMap_; + }; + + GpuUserEventMap gpuUserEventMap_; + // id -> activity* + std::unordered_map activityMap_; + // cuda runtime id -> pytorch op id + // CUPTI provides a mechanism for correlating Cuda events to arbitrary + // external events, e.g.operator activities from PyTorch. + std::unordered_map cpuCorrelationMap_; + // CUDA runtime <-> GPU Activity + std::unordered_map + correlatedCudaActivities_; + std::unordered_map userCorrelationMap_; + + // data structure to collect cuptiActivityFlushAll() latency overhead + struct profilerOverhead { + int64_t overhead; + int cntr; + }; + + bool isWarmupDone( + const std::chrono::time_point& now, + int64_t currentIter) const; + + bool isCollectionDone( + const std::chrono::time_point& now, + int64_t currentIter) const; + + void startTraceInternal( + const std::chrono::time_point& now); + + void stopTraceInternal( + const std::chrono::time_point& now); + + void processTraceInternal(ActivityLogger& logger); + + void resetInternal(); + + void finalizeTrace(const Config& config, ActivityLogger& logger); + + void configureChildProfilers(); + + // Process a single CPU trace + void processCpuTrace( + libkineto::CpuTraceBuffer& cpuTrace, + ActivityLogger& logger); + + // Create resource names for streams + inline void recordStream(int device, int id, const char* postfix) { + if (resourceInfo_.find({device, id}) == resourceInfo_.end()) { + resourceInfo_.emplace( + std::make_pair(device, id), + ActivityLogger::ResourceInfo( + device, id, id, fmt::format( + "stream {} {}", id, postfix))); + } + } + + // Record client trace span for subsequent lookups from activities + // Also creates a corresponding GPU-side span. + CpuGpuSpanPair& recordTraceSpan(TraceSpan& span, int gpuOpCount); + + // Returns true if net name is to be tracked for a specified number of + // iterations. + bool iterationTargetMatch(libkineto::CpuTraceBuffer& trace); + + // net name to id + int netId(const std::string& netName); + + const ITraceActivity* linkedActivity( + int32_t correlationId, + const std::unordered_map& correlationMap); + +#ifdef HAS_CUPTI + // Process generic CUPTI activity + void handleCuptiActivity(const CUpti_Activity* record, ActivityLogger* logger); + + // Process specific GPU activity types + void updateGpuNetSpan(const ITraceActivity& gpuOp); + bool outOfRange(const ITraceActivity& act); + void handleCorrelationActivity( + const CUpti_ActivityExternalCorrelation* correlation); + void handleRuntimeActivity( + const CUpti_ActivityAPI* activity, ActivityLogger* logger); + void handleOverheadActivity( + const CUpti_ActivityOverhead* activity, ActivityLogger* logger); + void handleGpuActivity(const ITraceActivity& act, + ActivityLogger* logger); + template + void handleGpuActivity(const T* act, ActivityLogger* logger); +#endif // HAS_CUPTI + + void resetTraceData(); + + void addOverheadSample(profilerOverhead& counter, int64_t overhead) { + counter.overhead += overhead; + counter.cntr++; + } + int64_t getOverhead(const profilerOverhead& counter) { + if (counter.cntr == 0) { + return 0; + } + return counter.overhead / counter.cntr; + } + + void checkTimestampOrder(const ITraceActivity* act1); + + // On-demand request configuration + std::unique_ptr config_; + + // Logger used during trace processing + ActivityLogger* logger_; + + // Calls to CUPTI is encapsulated behind this interface +#ifdef HAS_ROCTRACER + RoctracerActivityApi& cupti_; // Design failure here +#else + CuptiActivityApi& cupti_; +#endif + + enum class RunloopState { + WaitForRequest, + Warmup, + CollectTrace, + ProcessTrace + }; + + // Start and end time used for triggering and stopping profiling + std::chrono::time_point profileStartTime_; + std::chrono::time_point profileEndTime_; + int64_t profileStartIter_ = -1, profileEndIter_ = -1; + + + // All recorded trace spans, both CPU and GPU + // Trace Id -> list of iterations. + // Using map of lists for the iterator semantics, since we are recording + // pointers to the elements in this structure. + std::map> traceSpans_; + + // Maintain a map of client trace activity to trace span. + // Maps correlation id -> TraceSpan* held by traceSpans_. + using ActivityTraceMap = std::unordered_map; + ActivityTraceMap clientActivityTraceMap_; + + // Cache thread names and system thread ids for pthread ids, + // and stream ids for GPU streams + std::map< + std::pair, + ActivityLogger::ResourceInfo> resourceInfo_; + + std::vector overheadInfo_; + + // the overhead to flush the activity buffer + profilerOverhead flushOverhead_; + // the overhead to enable/disable activity tracking + profilerOverhead setupOverhead_; + + bool cpuOnly_{false}; + + // *************************************************************************** + // Below state is shared with external threads. + // These need to either be atomic, accessed under lock or only used + // by external threads in separate runloop phases from the profiler thread. + // *************************************************************************** + + // Mutex to protect non-atomic access to below state + std::mutex mutex_; + + // Runloop phase + std::atomic currentRunloopState_{RunloopState::WaitForRequest}; + + // Keep track of the start time of the first net in the current trace. + // This is only relevant to Caffe2 as PyTorch does not have nets. + // All CUDA events before this time will be removed + // Can be written by external threads during collection. + int64_t captureWindowStartTime_{0}; + // Similarly, all CUDA API events after the last net event will be removed + int64_t captureWindowEndTime_{0}; + + // span name -> iteration count + std::map iterationCountMap_; + // Flag used to stop tracing from external api callback. + // Needs to be atomic since it's set from a different thread. + std::atomic_bool stopCollection_{false}; + + // Buffers where trace data is stored + std::unique_ptr traceBuffers_; + + // Trace metadata + std::unordered_map metadata_; + + // child activity profilers + std::vector> profilers_; + + // a vector of active profiler plugin sessions + std::vector> sessions_; + + // LoggerCollector to collect all LOGs during the trace +#if !USE_GOOGLE_LOG + std::unique_ptr loggerCollectorMetadata_; +#endif // !USE_GOOGLE_LOG +}; + +} // namespace KINETO_NAMESPACE diff --git a/tb_plugins/profiling/libkineto/src/CuptiCallbackApi.cpp b/tb_plugins/profiling/libkineto/src/CuptiCallbackApi.cpp new file mode 100644 index 000000000..187600399 --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/CuptiCallbackApi.cpp @@ -0,0 +1,260 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#include "CuptiCallbackApi.h" + +#include +#include +#include +#include +#include + +#ifdef HAS_CUPTI +#include "cupti_call.h" +#endif +#include "Logger.h" + + +namespace KINETO_NAMESPACE { + +// limit on number of handles per callback type +constexpr size_t MAX_CB_FNS_PER_CB = 8; + +// Reader Writer lock types +using ReaderWriterLock = std::shared_timed_mutex; +using ReaderLockGuard = std::shared_lock; +using WriteLockGuard = std::unique_lock; + +static ReaderWriterLock callbackLock_; + +/* Callback Table : + * Overall goal of the design is to optimize the lookup of function + * pointers. The table is structured at two levels and the leaf + * elements in the table are std::list to enable fast access/inserts/deletes + * + * | + * -> cb id 0 -> std::list of callbacks + * ... + * -> cb id n -> std::list of callbacks + * | + * ... + * CallbackTable is the finaly table type above + * See type declrartions in header file. + */ + + +/* callback_switchboard : is the global callback handler we register + * with CUPTI. The goal is to make it as efficient as possible + * to re-direct to the registered callback(s). + * + * Few things to care about : + * a) use if/then switches rather than map/hash structures + * b) avoid dynamic memory allocations + * c) be aware of locking overheads + */ +#ifdef HAS_CUPTI +static void CUPTIAPI callback_switchboard( +#else +static void callback_switchboard( +#endif + void* /* unused */, + CUpti_CallbackDomain domain, + CUpti_CallbackId cbid, + const CUpti_CallbackData* cbInfo) { + + // below statement is likey going to call a mutex + // on the singleton access + CuptiCallbackApi::singleton().__callback_switchboard( + domain, cbid, cbInfo); +} + + +void CuptiCallbackApi::__callback_switchboard( + CUpti_CallbackDomain domain, + CUpti_CallbackId cbid, + const CUpti_CallbackData* cbInfo) { + VLOG(0) << "Callback: domain = " << domain << ", cbid = " << cbid; + CallbackList *cblist = nullptr; + + switch (domain) { + + // add the fastest path for kernel launch callbacks + // as these are the most frequent ones + case CUPTI_CB_DOMAIN_RUNTIME_API: + switch (cbid) { + case CUPTI_RUNTIME_TRACE_CBID_cudaLaunchKernel_v7000: + cblist = &callbacks_.runtime[ + CUDA_LAUNCH_KERNEL - __RUNTIME_CB_DOMAIN_START]; + break; + default: + break; + } + break; + + case CUPTI_CB_DOMAIN_RESOURCE: + switch (cbid) { + case CUPTI_CBID_RESOURCE_CONTEXT_CREATED: + cblist = &callbacks_.resource[ + RESOURCE_CONTEXT_CREATED - __RESOURCE_CB_DOMAIN_START]; + break; + case CUPTI_CBID_RESOURCE_CONTEXT_DESTROY_STARTING: + cblist = &callbacks_.resource[ + RESOURCE_CONTEXT_DESTROYED - __RESOURCE_CB_DOMAIN_START]; + break; + default: + break; + } + break; + + default: + return; + } + + // ignore callbacks that are not handled + if (cblist == nullptr) { + return; + } + + // make a copy of the callback list so we avoid holding lock + // in common case this should be just one func pointer copy + std::array callbacks; + int num_cbs = 0; + { + ReaderLockGuard rl(callbackLock_); + int i = 0; + for (auto it = cblist->begin(); + it != cblist->end() && i < MAX_CB_FNS_PER_CB; + it++, i++) { + callbacks[i] = *it; + } + num_cbs = i; + } + + for (int i = 0; i < num_cbs; i++) { + auto fn = callbacks[i]; + fn(domain, cbid, cbInfo); + } +} + +CuptiCallbackApi& CuptiCallbackApi::singleton() { + static CuptiCallbackApi instance; + return instance; +} + +CuptiCallbackApi::CuptiCallbackApi() { +#ifdef HAS_CUPTI + lastCuptiStatus_ = CUPTI_ERROR_UNKNOWN; + lastCuptiStatus_ = CUPTI_CALL_NOWARN( + cuptiSubscribe(&subscriber_, + (CUpti_CallbackFunc)callback_switchboard, + nullptr)); + + initSuccess_ = (lastCuptiStatus_ == CUPTI_SUCCESS); +#endif +} + +CuptiCallbackApi::CallbackList* CuptiCallbackApi::CallbackTable::lookup( + CUpti_CallbackDomain domain, CuptiCallBackID cbid) { + size_t idx; + + switch (domain) { + + case CUPTI_CB_DOMAIN_RESOURCE: + assert(cbid >= __RESOURCE_CB_DOMAIN_START); + assert(cbid < __RESOURCE_CB_DOMAIN_END); + idx = cbid - __RESOURCE_CB_DOMAIN_START; + return &resource.at(idx); + + case CUPTI_CB_DOMAIN_RUNTIME_API: + assert(cbid >= __RUNTIME_CB_DOMAIN_START); + assert(cbid < __RUNTIME_CB_DOMAIN_END); + idx = cbid - __RUNTIME_CB_DOMAIN_START; + return &runtime.at(idx); + + default: + LOG(WARNING) << " Unsupported callback domain : " << domain; + return nullptr; + } +} + +bool CuptiCallbackApi::registerCallback( + CUpti_CallbackDomain domain, + CuptiCallBackID cbid, + CuptiCallbackFn cbfn) { + CallbackList* cblist = callbacks_.lookup(domain, cbid); + + if (!cblist) { + LOG(WARNING) << "Could not register callback -- domain = " << domain + << " callback id = " << cbid; + return false; + } + + // avoid duplicates + auto it = std::find(cblist->begin(), cblist->end(), cbfn); + if (it != cblist->end()) { + LOG(WARNING) << "Adding duplicate callback -- domain = " << domain + << " callback id = " << cbid; + return true; + } + + if (cblist->size() == MAX_CB_FNS_PER_CB) { + LOG(WARNING) << "Already registered max callback -- domain = " << domain + << " callback id = " << cbid; + } + + WriteLockGuard wl(callbackLock_); + cblist->push_back(cbfn); + return true; +} + +bool CuptiCallbackApi::deleteCallback( + CUpti_CallbackDomain domain, + CuptiCallBackID cbid, + CuptiCallbackFn cbfn) { + CallbackList* cblist = callbacks_.lookup(domain, cbid); + if (!cblist) { + LOG(WARNING) << "Attempting to remove unsupported callback -- domain = " << domain + << " callback id = " << cbid; + return false; + } + + // Locks are not required here as + // https://en.cppreference.com/w/cpp/container/list/erase + // "References and iterators to the erased elements are invalidated. + // Other references and iterators are not affected." + auto it = std::find(cblist->begin(), cblist->end(), cbfn); + if (it == cblist->end()) { + LOG(WARNING) << "Could not find callback to remove -- domain = " << domain + << " callback id = " << cbid; + return false; + } + + WriteLockGuard wl(callbackLock_); + cblist->erase(it); + return true; +} + +bool CuptiCallbackApi::enableCallback( + CUpti_CallbackDomain domain, CUpti_CallbackId cbid) { +#ifdef HAS_CUPTI + if (initSuccess_) { + lastCuptiStatus_ = CUPTI_CALL_NOWARN( + cuptiEnableCallback(1, subscriber_, domain, cbid)); + return (lastCuptiStatus_ == CUPTI_SUCCESS); + } +#endif + return false; +} + +bool CuptiCallbackApi::disableCallback( + CUpti_CallbackDomain domain, CUpti_CallbackId cbid) { +#ifdef HAS_CUPTI + if (initSuccess_) { + lastCuptiStatus_ = CUPTI_CALL_NOWARN( + cuptiEnableCallback(0, subscriber_, domain, cbid)); + return (lastCuptiStatus_ == CUPTI_SUCCESS); + } +#endif + return false; +} + +} // namespace KINETO_NAMESPACE diff --git a/tb_plugins/profiling/libkineto/src/CuptiCallbackApi.h b/tb_plugins/profiling/libkineto/src/CuptiCallbackApi.h new file mode 100644 index 000000000..4526f3750 --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/CuptiCallbackApi.h @@ -0,0 +1,130 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#pragma once + +#include +#ifdef HAS_CUPTI +#include +#endif +#include +#include +#include +#include +#include + +// TODO(T90238193) +// @lint-ignore-every CLANGTIDY facebook-hte-RelativeInclude +#include "CuptiCallbackApiMock.h" + +namespace KINETO_NAMESPACE { + +using namespace libkineto; + + +/* CuptiCallbackApi : Provides an abstraction over CUPTI callback + * interface. This enables various callback functions to be registered + * with this class. The class registers a global callback handler that + * redirects to the respective callbacks. + * + * Note: one design choice we made is to only support simple function pointers + * in order to speed up the implementation for fast path. + */ + +using CuptiCallbackFn = void(*)( + CUpti_CallbackDomain domain, + CUpti_CallbackId cbid, + const CUpti_CallbackData* cbInfo); + + +class CuptiCallbackApi { + + public: + + /* Global list of supported callback ids + * use the class namespace to avoid confusing with CUPTI enums*/ + enum CuptiCallBackID { + CUDA_LAUNCH_KERNEL = 0, + // can possibly support more callback ids per domain + // + __RUNTIME_CB_DOMAIN_START = CUDA_LAUNCH_KERNEL, + + // Callbacks under Resource CB domain + RESOURCE_CONTEXT_CREATED, + RESOURCE_CONTEXT_DESTROYED, + + __RUNTIME_CB_DOMAIN_END = RESOURCE_CONTEXT_CREATED, + __RESOURCE_CB_DOMAIN_START = RESOURCE_CONTEXT_CREATED, + + __RESOURCE_CB_DOMAIN_END = RESOURCE_CONTEXT_DESTROYED + 1, + }; + + + CuptiCallbackApi(const CuptiCallbackApi&) = delete; + CuptiCallbackApi& operator=(const CuptiCallbackApi&) = delete; + + static CuptiCallbackApi& singleton(); + + bool initSuccess() const { + return initSuccess_; + } + +#ifdef HAS_CUPTI + CUptiResult getCuptiStatus() const { + return lastCuptiStatus_; + } +#endif + + bool registerCallback( + CUpti_CallbackDomain domain, + CuptiCallBackID cbid, + CuptiCallbackFn cbfn); + + // returns false if callback was not found + bool deleteCallback( + CUpti_CallbackDomain domain, + CuptiCallBackID cbid, + CuptiCallbackFn cbfn); + + bool enableCallback(CUpti_CallbackDomain domain, CUpti_CallbackId cbid); + bool disableCallback(CUpti_CallbackDomain domain, CUpti_CallbackId cbid); + + + // Please do not use this method. This has to be exposed as public + // so it is accessible from the callback handler + void __callback_switchboard( + CUpti_CallbackDomain domain, + CUpti_CallbackId cbid, + const CUpti_CallbackData* cbInfo); + + private: + + explicit CuptiCallbackApi(); + + // For callback table design overview see the .cpp file + using CallbackList = std::list; + + // level 2 tables sizes are known at compile time + constexpr static size_t RUNTIME_CB_DOMAIN_SIZE + = (__RUNTIME_CB_DOMAIN_END - __RUNTIME_CB_DOMAIN_START); + + constexpr static size_t RESOURCE_CB_DOMAIN_SIZE + = (__RESOURCE_CB_DOMAIN_END - __RESOURCE_CB_DOMAIN_START); + + // level 1 table is a struct + struct CallbackTable { + std::array runtime; + std::array resource; + + CallbackList* lookup(CUpti_CallbackDomain domain, CuptiCallBackID cbid); + }; + + CallbackTable callbacks_; + bool initSuccess_ = false; + +#ifdef HAS_CUPTI + CUptiResult lastCuptiStatus_; + CUpti_SubscriberHandle subscriber_; +#endif +}; + +} // namespace KINETO_NAMESPACE diff --git a/tb_plugins/profiling/libkineto/src/CuptiCallbackApiMock.h b/tb_plugins/profiling/libkineto/src/CuptiCallbackApiMock.h new file mode 100644 index 000000000..fd5126727 --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/CuptiCallbackApiMock.h @@ -0,0 +1,32 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#pragma once + +// Provides data structures to mock CUPTI Callback API +#ifndef HAS_CUPTI + +enum CUpti_CallbackDomain { + CUPTI_CB_DOMAIN_RESOURCE, + CUPTI_CB_DOMAIN_RUNTIME_API, +}; +enum CUpti_CallbackId { + CUPTI_RUNTIME_TRACE_CBID_cudaLaunchKernel_v7000, + CUPTI_CBID_RESOURCE_CONTEXT_CREATED, + CUPTI_CBID_RESOURCE_CONTEXT_DESTROY_STARTING, +}; + +using CUcontext = void*; + +struct CUpti_ResourceData { + CUcontext context; +}; + +constexpr int CUPTI_API_ENTER = 0; +constexpr int CUPTI_API_EXIT = 0; + +struct CUpti_CallbackData { + CUcontext context; + const char* symbolName; + int callbackSite; +}; +#endif // HAS_CUPTI diff --git a/tb_plugins/profiling/libkineto/src/CuptiEventApi.cpp b/tb_plugins/profiling/libkineto/src/CuptiEventApi.cpp new file mode 100644 index 000000000..7f1d48c1d --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/CuptiEventApi.cpp @@ -0,0 +1,112 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#include "CuptiEventApi.h" + +#include + +#include "Logger.h" +#include "cupti_call.h" + +using namespace std::chrono; +using std::vector; + +namespace KINETO_NAMESPACE { + +CuptiEventApi::CuptiEventApi(CUcontext context) + : context_(context) { + CUPTI_CALL(cuptiGetDeviceId(context_, (uint32_t*)&device_)); +} + +CUpti_EventGroupSets* CuptiEventApi::createGroupSets( + vector& ids) { + CUpti_EventGroupSets* group_sets = nullptr; + CUptiResult res = CUPTI_CALL(cuptiEventGroupSetsCreate( + context_, sizeof(CUpti_EventID) * ids.size(), ids.data(), &group_sets)); + + if (res != CUPTI_SUCCESS || group_sets == nullptr) { + const char* errstr = nullptr; + CUPTI_CALL(cuptiGetResultString(res, &errstr)); + throw std::system_error(EINVAL, std::generic_category(), errstr); + } + + return group_sets; +} + +void CuptiEventApi::destroyGroupSets(CUpti_EventGroupSets* sets) { + CUPTI_CALL(cuptiEventGroupSetsDestroy(sets)); +} + +bool CuptiEventApi::setContinuousMode() { + // Avoid logging noise for CUPTI_ERROR_LEGACY_PROFILER_NOT_SUPPORTED + CUptiResult res = CUPTI_CALL_NOWARN(cuptiSetEventCollectionMode( + context_, CUPTI_EVENT_COLLECTION_MODE_CONTINUOUS)); + if (res == CUPTI_ERROR_LEGACY_PROFILER_NOT_SUPPORTED) { + return false; + } + // Log warning on other errors + CUPTI_CALL(res); + return (res == CUPTI_SUCCESS); +} + +void CuptiEventApi::enablePerInstance(CUpti_EventGroup eventGroup) { + uint32_t profile_all = 1; + CUPTI_CALL(cuptiEventGroupSetAttribute( + eventGroup, + CUPTI_EVENT_GROUP_ATTR_PROFILE_ALL_DOMAIN_INSTANCES, + sizeof(profile_all), + &profile_all)); +} + +uint32_t CuptiEventApi::instanceCount(CUpti_EventGroup eventGroup) { + uint32_t instance_count = 0; + size_t s = sizeof(instance_count); + CUPTI_CALL(cuptiEventGroupGetAttribute( + eventGroup, CUPTI_EVENT_GROUP_ATTR_INSTANCE_COUNT, &s, &instance_count)); + return instance_count; +} + +void CuptiEventApi::enableGroupSet(CUpti_EventGroupSet& set) { + CUptiResult res = CUPTI_CALL_NOWARN(cuptiEventGroupSetEnable(&set)); + if (res != CUPTI_SUCCESS) { + const char* errstr = nullptr; + CUPTI_CALL(cuptiGetResultString(res, &errstr)); + throw std::system_error(EIO, std::generic_category(), errstr); + } +} + +void CuptiEventApi::disableGroupSet(CUpti_EventGroupSet& set) { + CUPTI_CALL(cuptiEventGroupSetDisable(&set)); +} + +void CuptiEventApi::readEvent( + CUpti_EventGroup grp, + CUpti_EventID id, + vector& vals) { + size_t s = sizeof(int64_t) * vals.size(); + CUPTI_CALL(cuptiEventGroupReadEvent( + grp, + CUPTI_EVENT_READ_FLAG_NONE, + id, + &s, + reinterpret_cast(vals.data()))); +} + +vector CuptiEventApi::eventsInGroup(CUpti_EventGroup grp) { + uint32_t group_size = 0; + size_t s = sizeof(group_size); + CUPTI_CALL(cuptiEventGroupGetAttribute( + grp, CUPTI_EVENT_GROUP_ATTR_NUM_EVENTS, &s, &group_size)); + size_t events_size = group_size * sizeof(CUpti_EventID); + vector res(group_size); + CUPTI_CALL(cuptiEventGroupGetAttribute( + grp, CUPTI_EVENT_GROUP_ATTR_EVENTS, &events_size, res.data())); + return res; +} + +CUpti_EventID CuptiEventApi::eventId(const std::string& name) { + CUpti_EventID id{0}; + CUPTI_CALL(cuptiEventGetIdFromName(device_, name.c_str(), &id)); + return id; +} + +} // namespace KINETO_NAMESPACE diff --git a/tb_plugins/profiling/libkineto/src/CuptiEventApi.h b/tb_plugins/profiling/libkineto/src/CuptiEventApi.h new file mode 100644 index 000000000..79610f93f --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/CuptiEventApi.h @@ -0,0 +1,49 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#pragma once + +#include +#include +#include + +namespace KINETO_NAMESPACE { + +// C++ interface to CUPTI Events C API. +// Virtual methods are here mainly to allow easier testing. +class CuptiEventApi { + public: + explicit CuptiEventApi(CUcontext context_); + virtual ~CuptiEventApi() {} + + CUdevice device() { + return device_; + } + + virtual CUpti_EventGroupSets* createGroupSets( + std::vector& ids); + virtual void destroyGroupSets(CUpti_EventGroupSets* sets); + + virtual bool setContinuousMode(); + + virtual void enablePerInstance(CUpti_EventGroup eventGroup); + virtual uint32_t instanceCount(CUpti_EventGroup eventGroup); + + virtual void enableGroupSet(CUpti_EventGroupSet& set); + virtual void disableGroupSet(CUpti_EventGroupSet& set); + + virtual void + readEvent(CUpti_EventGroup g, CUpti_EventID id, std::vector& vals); + virtual std::vector eventsInGroup(CUpti_EventGroup g); + + virtual CUpti_EventID eventId(const std::string& name); + + protected: + // Unit testing + CuptiEventApi() : context_(nullptr), device_(0) {} + + private: + CUcontext context_; + CUdevice device_; +}; + +} // namespace KINETO_NAMESPACE diff --git a/tb_plugins/profiling/libkineto/src/CuptiMetricApi.cpp b/tb_plugins/profiling/libkineto/src/CuptiMetricApi.cpp new file mode 100644 index 000000000..36401e743 --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/CuptiMetricApi.cpp @@ -0,0 +1,107 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#include "CuptiMetricApi.h" + +#include + +#include "Logger.h" +#include "cupti_call.h" + +using namespace std::chrono; +using std::vector; + +namespace KINETO_NAMESPACE { + +CUpti_MetricID CuptiMetricApi::idFromName(const std::string& name) { + CUpti_MetricID metric_id{~0u}; + CUptiResult res = + CUPTI_CALL(cuptiMetricGetIdFromName(device_, name.c_str(), &metric_id)); + if (res == CUPTI_ERROR_INVALID_METRIC_NAME) { + LOG(WARNING) << "Invalid metric name: " << name; + } + return metric_id; +} + +// Return a map of event IDs and names for a given metric id. +// Note that many events don't have a name. In that case the name will +// be set to the empty string. +std::map CuptiMetricApi::events( + CUpti_MetricID metric_id) { + uint32_t num_events = 0; + CUPTI_CALL(cuptiMetricGetNumEvents(metric_id, &num_events)); + vector ids(num_events); + size_t array_size = num_events * sizeof(CUpti_EventID); + CUPTI_CALL(cuptiMetricEnumEvents(metric_id, &array_size, ids.data())); + std::map res; + for (CUpti_EventID id : ids) { + // Attempt to lookup name from CUPTI + constexpr size_t kMaxEventNameLength = 64; + char cupti_name[kMaxEventNameLength]; + size_t size = kMaxEventNameLength; + CUPTI_CALL( + cuptiEventGetAttribute(id, CUPTI_EVENT_ATTR_NAME, &size, cupti_name)); + cupti_name[kMaxEventNameLength - 1] = 0; + + // CUPTI "helpfully" returns "event_name" when the event is unnamed. + if (size > 0 && strcmp(cupti_name, "event_name") != 0) { + res.emplace(id, cupti_name); + } else { + res.emplace(id, ""); + } + } + return res; +} + +CUpti_MetricValueKind CuptiMetricApi::valueKind(CUpti_MetricID metric) { + CUpti_MetricValueKind res{CUPTI_METRIC_VALUE_KIND_FORCE_INT}; + size_t value_kind_size = sizeof(res); + CUPTI_CALL(cuptiMetricGetAttribute( + metric, CUPTI_METRIC_ATTR_VALUE_KIND, &value_kind_size, &res)); + return res; +} + +CUpti_MetricEvaluationMode CuptiMetricApi::evaluationMode( + CUpti_MetricID metric) { + CUpti_MetricEvaluationMode eval_mode{ + CUPTI_METRIC_EVALUATION_MODE_PER_INSTANCE}; + size_t eval_mode_size = sizeof(eval_mode); + CUPTI_CALL(cuptiMetricGetAttribute( + metric, CUPTI_METRIC_ATTR_EVALUATION_MODE, &eval_mode_size, &eval_mode)); + return eval_mode; +} + +// FIXME: Consider caching value kind here +SampleValue CuptiMetricApi::calculate( + CUpti_MetricID metric, + CUpti_MetricValueKind kind, + vector& events, + vector& values, + int64_t duration) { + CUpti_MetricValue metric_value; + CUPTI_CALL(cuptiMetricGetValue( + device_, + metric, + events.size() * sizeof(CUpti_EventID), + events.data(), + values.size() * sizeof(int64_t), + reinterpret_cast(values.data()), + duration, + &metric_value)); + + switch (kind) { + case CUPTI_METRIC_VALUE_KIND_DOUBLE: + case CUPTI_METRIC_VALUE_KIND_PERCENT: + return SampleValue(metric_value.metricValueDouble); + case CUPTI_METRIC_VALUE_KIND_UINT64: + case CUPTI_METRIC_VALUE_KIND_INT64: + case CUPTI_METRIC_VALUE_KIND_THROUGHPUT: + return SampleValue(metric_value.metricValueUint64); + case CUPTI_METRIC_VALUE_KIND_UTILIZATION_LEVEL: + return SampleValue((int)metric_value.metricValueUtilizationLevel); + default: + assert(false); + } + return SampleValue(-1); +} + +} // namespace KINETO_NAMESPACE diff --git a/tb_plugins/profiling/libkineto/src/CuptiMetricApi.h b/tb_plugins/profiling/libkineto/src/CuptiMetricApi.h new file mode 100644 index 000000000..f45d38cd6 --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/CuptiMetricApi.h @@ -0,0 +1,38 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#pragma once + +#include + +#include +#include + +#include "SampleListener.h" + +namespace KINETO_NAMESPACE { + +// C++ interface to CUPTI Metrics C API. +// Virtual methods are here mainly to allow easier testing. +class CuptiMetricApi { + public: + explicit CuptiMetricApi(CUdevice device) : device_(device) {} + virtual ~CuptiMetricApi() {} + + virtual CUpti_MetricID idFromName(const std::string& name); + virtual std::map events(CUpti_MetricID metric_id); + + virtual CUpti_MetricValueKind valueKind(CUpti_MetricID metric); + virtual CUpti_MetricEvaluationMode evaluationMode(CUpti_MetricID metric); + + virtual SampleValue calculate( + CUpti_MetricID metric, + CUpti_MetricValueKind kind, + std::vector& events, + std::vector& values, + int64_t duration); + + private: + CUdevice device_; +}; + +} // namespace KINETO_NAMESPACE diff --git a/tb_plugins/profiling/libkineto/src/CuptiNvPerfMetric.cpp b/tb_plugins/profiling/libkineto/src/CuptiNvPerfMetric.cpp new file mode 100644 index 000000000..d1b08ab2c --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/CuptiNvPerfMetric.cpp @@ -0,0 +1,504 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#ifdef HAS_CUPTI +#include +#if defined(CUDART_VERSION) && CUDART_VERSION > 10000 && CUDART_VERSION < 11040 +#include +#include +#include +#endif // cuda version > 10.00 and < 11.04 +#endif // HAS_CUPTI + +// TODO(T90238193) +// @lint-ignore-every CLANGTIDY facebook-hte-RelativeInclude +#include "ScopeExit.h" +#include "CuptiNvPerfMetric.h" +#include "Logger.h" + +namespace KINETO_NAMESPACE { + +// Add a namespace to isolate these utility functions that are only +// going to be used by the CuptiRangeProfiler. These included calls +// to NVIDIA PerfWorks APIs. +namespace nvperf { + + +// Largely based on NVIDIA sample code provided with CUDA release +// files Metric.cpp and Eval.cpp + +// ------------------------------------------------- +// Metric and Counter Data Configuration +// ------------------------------------------------- + + +// Note: Be carful before modifying the code below. There is a specific +// sequence one needs to follow to program the metrics else things may +// stop working. We tried to keep the flow consistent with the example +// code from NVIDIA. Since most of the programmability comes from +// the CUPTI profiler metric names this should be okay. + +// Only supported on CUDA RT Version between 10.0 and 11.04. +// After CUDA RT 11.04, the structure has changed. +// TODO update the structure NVPA_RawMetricsConfig to support 11.04 +#if defined(CUDART_VERSION) && CUDART_VERSION > 10000 && CUDART_VERSION < 11040 + +bool getRawMetricRequests( + NVPA_MetricsContext* metricsContext, + std::vector metricNames, + std::vector& rawMetricsDeps, + std::vector& rawMetricRequests) { + bool isolated = true; + /* Bug in collection with collection of metrics without instances, keep it + * to true*/ + bool keepInstances = true; + + for (const auto& metricName : metricNames) { + + NVPW_MetricsContext_GetMetricProperties_Begin_Params + getMetricPropertiesBeginParams = { + NVPW_MetricsContext_GetMetricProperties_Begin_Params_STRUCT_SIZE, nullptr}; + getMetricPropertiesBeginParams.pMetricsContext = metricsContext; + getMetricPropertiesBeginParams.pMetricName = metricName.c_str(); + + if (!NVPW_CALL( + NVPW_MetricsContext_GetMetricProperties_Begin( + &getMetricPropertiesBeginParams))) { + return false; + } + + for (const char** metricDepsIt = + getMetricPropertiesBeginParams.ppRawMetricDependencies; + *metricDepsIt; + ++metricDepsIt) { + rawMetricsDeps.push_back(*metricDepsIt); + } + + NVPW_MetricsContext_GetMetricProperties_End_Params + getMetricPropertiesEndParams = { + NVPW_MetricsContext_GetMetricProperties_End_Params_STRUCT_SIZE, nullptr}; + getMetricPropertiesEndParams.pMetricsContext = metricsContext; + + if (!NVPW_CALL(NVPW_MetricsContext_GetMetricProperties_End( + &getMetricPropertiesEndParams))) { + return false; + } + } + + for (const auto& rawMetricName : rawMetricsDeps) { + NVPA_RawMetricRequest metricRequest = {NVPA_RAW_METRIC_REQUEST_STRUCT_SIZE, nullptr}; + metricRequest.pMetricName = rawMetricName.c_str(); + metricRequest.isolated = isolated; + metricRequest.keepInstances = keepInstances; + rawMetricRequests.push_back(metricRequest); + VLOG(1) << "Adding raw metric struct : raw metric = " << rawMetricName + << " isolated = " << isolated << " keepinst = " << keepInstances; + } + + if (rawMetricRequests.size() == 0) { + LOG(WARNING) << "CUPTI Profiler was unable to configure any metrics"; + return false; + } + return true; +} + +// Setup CUPTI Profiler Config Image +bool getProfilerConfigImage( + const std::string& chipName, + const std::vector& metricNames, + std::vector& configImage, + const uint8_t* counterAvailabilityImage) { + + NVPW_CUDA_MetricsContext_Create_Params metricsContextCreateParams = { + NVPW_CUDA_MetricsContext_Create_Params_STRUCT_SIZE, nullptr}; + metricsContextCreateParams.pChipName = chipName.c_str(); + + if (!NVPW_CALL( + NVPW_CUDA_MetricsContext_Create(&metricsContextCreateParams))) { + return false; + } + + NVPW_MetricsContext_Destroy_Params metricsContextDestroyParams = { + NVPW_MetricsContext_Destroy_Params_STRUCT_SIZE, nullptr}; + metricsContextDestroyParams.pMetricsContext = + metricsContextCreateParams.pMetricsContext; + + SCOPE_EXIT([&]() { + NVPW_MetricsContext_Destroy( + (NVPW_MetricsContext_Destroy_Params*)&metricsContextDestroyParams); + }); + + // Get all raw metrics required for given metricNames list + std::vector rawMetricRequests; + + // note: we need a variable at this functions scope to hold the string + // pointers for underlying C char arrays. + std::vector rawMetricDeps; + + if (!getRawMetricRequests( + metricsContextCreateParams.pMetricsContext, + metricNames, + rawMetricDeps, + rawMetricRequests)) { + return false; + } + + NVPA_RawMetricsConfigOptions metricsConfigOptions = { + NVPA_RAW_METRICS_CONFIG_OPTIONS_STRUCT_SIZE, nullptr}; + metricsConfigOptions.activityKind = NVPA_ACTIVITY_KIND_PROFILER; + metricsConfigOptions.pChipName = chipName.c_str(); + NVPA_RawMetricsConfig* rawMetricsConfig; + if (!NVPW_CALL( + NVPA_RawMetricsConfig_Create( + &metricsConfigOptions, &rawMetricsConfig))) { + return false; + } + + // TODO check if this is required + if (counterAvailabilityImage) { + NVPW_RawMetricsConfig_SetCounterAvailability_Params + setCounterAvailabilityParams = { + NVPW_RawMetricsConfig_SetCounterAvailability_Params_STRUCT_SIZE, nullptr}; + setCounterAvailabilityParams.pRawMetricsConfig = rawMetricsConfig; + setCounterAvailabilityParams.pCounterAvailabilityImage = + counterAvailabilityImage; + if (!NVPW_CALL( + NVPW_RawMetricsConfig_SetCounterAvailability( + &setCounterAvailabilityParams))) { + return false; + } + } + + NVPW_RawMetricsConfig_Destroy_Params rawMetricsConfigDestroyParams = { + NVPW_RawMetricsConfig_Destroy_Params_STRUCT_SIZE, nullptr}; + rawMetricsConfigDestroyParams.pRawMetricsConfig = rawMetricsConfig; + SCOPE_EXIT([&]() { + NVPW_RawMetricsConfig_Destroy( + (NVPW_RawMetricsConfig_Destroy_Params*)&rawMetricsConfigDestroyParams); + }); + + // Start a Raw Metric Pass group + NVPW_RawMetricsConfig_BeginPassGroup_Params beginPassGroupParams = { + NVPW_RawMetricsConfig_BeginPassGroup_Params_STRUCT_SIZE, nullptr}; + beginPassGroupParams.pRawMetricsConfig = rawMetricsConfig; + if (!NVPW_CALL( + NVPW_RawMetricsConfig_BeginPassGroup(&beginPassGroupParams))) { + return false; + } + + // Add all raw metrics + NVPW_RawMetricsConfig_AddMetrics_Params addMetricsParams = { + NVPW_RawMetricsConfig_AddMetrics_Params_STRUCT_SIZE, nullptr}; + addMetricsParams.pRawMetricsConfig = rawMetricsConfig; + addMetricsParams.pRawMetricRequests = rawMetricRequests.data(); + addMetricsParams.numMetricRequests = rawMetricRequests.size(); + if (!NVPW_CALL( + NVPW_RawMetricsConfig_AddMetrics(&addMetricsParams))) { + return false; + } + + // End pass group + NVPW_RawMetricsConfig_EndPassGroup_Params endPassGroupParams = { + NVPW_RawMetricsConfig_EndPassGroup_Params_STRUCT_SIZE, nullptr}; + endPassGroupParams.pRawMetricsConfig = rawMetricsConfig; + if (!NVPW_CALL( + NVPW_RawMetricsConfig_EndPassGroup(&endPassGroupParams))) { + return false; + } + + // Setup Config Image generation + NVPW_RawMetricsConfig_GenerateConfigImage_Params generateConfigImageParams = { + NVPW_RawMetricsConfig_GenerateConfigImage_Params_STRUCT_SIZE, nullptr}; + generateConfigImageParams.pRawMetricsConfig = rawMetricsConfig; + if (!NVPW_CALL( + NVPW_RawMetricsConfig_GenerateConfigImage(&generateConfigImageParams))) { + return false; + } + + // Get the Config Image size... nearly there + NVPW_RawMetricsConfig_GetConfigImage_Params getConfigImageParams = { + NVPW_RawMetricsConfig_GetConfigImage_Params_STRUCT_SIZE, nullptr}; + getConfigImageParams.pRawMetricsConfig = rawMetricsConfig; + getConfigImageParams.bytesAllocated = 0; + getConfigImageParams.pBuffer = nullptr; + if (!NVPW_CALL( + NVPW_RawMetricsConfig_GetConfigImage(&getConfigImageParams))) { + return false; + } + + configImage.resize(getConfigImageParams.bytesCopied); + + // Write the Config image binary + getConfigImageParams.bytesAllocated = configImage.size(); + getConfigImageParams.pBuffer = configImage.data(); + if (!NVPW_CALL( + NVPW_RawMetricsConfig_GetConfigImage(&getConfigImageParams))) { + return false; + } + + return true; +} + +bool getCounterDataPrefixImage( + const std::string& chipName, + const std::vector& metricNames, + std::vector& counterDataImagePrefix) { + + NVPW_CUDA_MetricsContext_Create_Params metricsContextCreateParams = { + NVPW_CUDA_MetricsContext_Create_Params_STRUCT_SIZE, nullptr}; + metricsContextCreateParams.pChipName = chipName.c_str(); + + if (!NVPW_CALL( + NVPW_CUDA_MetricsContext_Create(&metricsContextCreateParams))) { + return false; + } + + NVPW_MetricsContext_Destroy_Params metricsContextDestroyParams = { + NVPW_MetricsContext_Destroy_Params_STRUCT_SIZE, nullptr}; + metricsContextDestroyParams.pMetricsContext = + metricsContextCreateParams.pMetricsContext; + + + SCOPE_EXIT([&]() { + NVPW_MetricsContext_Destroy( + (NVPW_MetricsContext_Destroy_Params*)&metricsContextDestroyParams); + }); + + // Get all raw metrics required for given metricNames list + std::vector rawMetricRequests; + + // note: we need a variable at this functions scope to hold the string + // pointers for underlying C char arrays. + std::vector rawMetricDeps; + + if (!getRawMetricRequests( + metricsContextCreateParams.pMetricsContext, + metricNames, + rawMetricDeps, + rawMetricRequests)) { + return false; + } + + // Setup Counter Data builder + NVPW_CounterDataBuilder_Create_Params counterDataBuilderCreateParams = { + NVPW_CounterDataBuilder_Create_Params_STRUCT_SIZE, nullptr}; + counterDataBuilderCreateParams.pChipName = chipName.c_str(); + if (!NVPW_CALL( + NVPW_CounterDataBuilder_Create(&counterDataBuilderCreateParams))) { + return false; + } + + NVPW_CounterDataBuilder_Destroy_Params counterDataBuilderDestroyParams = { + NVPW_CounterDataBuilder_Destroy_Params_STRUCT_SIZE, nullptr}; + counterDataBuilderDestroyParams.pCounterDataBuilder = + counterDataBuilderCreateParams.pCounterDataBuilder; + SCOPE_EXIT([&]() { + NVPW_CounterDataBuilder_Destroy(( + NVPW_CounterDataBuilder_Destroy_Params*)&counterDataBuilderDestroyParams); + }); + + // Add metrics to counter data image prefix + NVPW_CounterDataBuilder_AddMetrics_Params addMetricsParams = { + NVPW_CounterDataBuilder_AddMetrics_Params_STRUCT_SIZE, nullptr}; + addMetricsParams.pCounterDataBuilder = + counterDataBuilderCreateParams.pCounterDataBuilder; + addMetricsParams.pRawMetricRequests = rawMetricRequests.data(); + addMetricsParams.numMetricRequests = rawMetricRequests.size(); + if (!NVPW_CALL( + NVPW_CounterDataBuilder_AddMetrics(&addMetricsParams))) { + return false; + } + + // Get image prefix size + NVPW_CounterDataBuilder_GetCounterDataPrefix_Params + getCounterDataPrefixParams = { + NVPW_CounterDataBuilder_GetCounterDataPrefix_Params_STRUCT_SIZE, nullptr}; + getCounterDataPrefixParams.pCounterDataBuilder = + counterDataBuilderCreateParams.pCounterDataBuilder; + getCounterDataPrefixParams.bytesAllocated = 0; + getCounterDataPrefixParams.pBuffer = nullptr; + if (!NVPW_CALL( + NVPW_CounterDataBuilder_GetCounterDataPrefix( + &getCounterDataPrefixParams))) { + return false; + } + + counterDataImagePrefix.resize(getCounterDataPrefixParams.bytesCopied); + + // Now write counter data image prefix + getCounterDataPrefixParams.bytesAllocated = counterDataImagePrefix.size(); + getCounterDataPrefixParams.pBuffer = counterDataImagePrefix.data(); + if (!NVPW_CALL( + NVPW_CounterDataBuilder_GetCounterDataPrefix( + &getCounterDataPrefixParams))) { + return false; + } + + return true; +} + +// ------------------------------------------------- +// Metric and Counter Evaluation Utilities +// ------------------------------------------------- + +std::string getRangeDescription( + const std::vector& counterDataImage, + int rangeIndex) { + std::vector descriptionPtrs; + + NVPW_Profiler_CounterData_GetRangeDescriptions_Params getRangeDescParams = { + NVPW_Profiler_CounterData_GetRangeDescriptions_Params_STRUCT_SIZE, nullptr}; + getRangeDescParams.pCounterDataImage = counterDataImage.data(); + getRangeDescParams.rangeIndex = rangeIndex; + + if (!NVPW_CALL( + NVPW_Profiler_CounterData_GetRangeDescriptions(&getRangeDescParams))) { + return ""; + } + + descriptionPtrs.resize(getRangeDescParams.numDescriptions); + getRangeDescParams.ppDescriptions = descriptionPtrs.data(); + + if (!NVPW_CALL( + NVPW_Profiler_CounterData_GetRangeDescriptions(&getRangeDescParams))) { + return ""; + } + + std::string rangeName; + + for (size_t i = 0; i < getRangeDescParams.numDescriptions; i++) { + if (i > 0) { + rangeName.append("/"); + } + rangeName.append(descriptionPtrs[i]); + } + return rangeName; +} + +CuptiProfilerResult evalMetricValues( + const std::string& chipName, + const std::vector& counterDataImage, + const std::vector& metricNames, + bool verbose) { + + if (!counterDataImage.size()) { + LOG(ERROR) << "Counter Data Image is empty!"; + return {}; + } + + NVPW_CUDA_MetricsContext_Create_Params metricsContextCreateParams = { + NVPW_CUDA_MetricsContext_Create_Params_STRUCT_SIZE, nullptr}; + metricsContextCreateParams.pChipName = chipName.c_str(); + if (!NVPW_CALL( + NVPW_CUDA_MetricsContext_Create(&metricsContextCreateParams))) { + return {}; + } + + NVPW_MetricsContext_Destroy_Params metricsContextDestroyParams = { + NVPW_MetricsContext_Destroy_Params_STRUCT_SIZE, nullptr}; + metricsContextDestroyParams.pMetricsContext = + metricsContextCreateParams.pMetricsContext; + SCOPE_EXIT([&]() { + NVPW_MetricsContext_Destroy( + (NVPW_MetricsContext_Destroy_Params*)&metricsContextDestroyParams); + }); + + NVPW_CounterData_GetNumRanges_Params getNumRangesParams = { + NVPW_CounterData_GetNumRanges_Params_STRUCT_SIZE, nullptr}; + getNumRangesParams.pCounterDataImage = counterDataImage.data(); + if (!NVPW_CALL( + NVPW_CounterData_GetNumRanges(&getNumRangesParams))) { + return {}; + } + + // TBD in the future support special chars in metric name + // for now these are default + const bool isolated = true; + + // API takes a 2D array of chars + std::vector metricNamePtrs; + + for (const auto& metric : metricNames) { + metricNamePtrs.push_back(metric.c_str()); + } + + CuptiProfilerResult result{ + .metricNames = metricNames}; + + for (size_t rangeIndex = 0; rangeIndex < getNumRangesParams.numRanges; + ++rangeIndex) { + + CuptiRangeMeasurement rangeData { + .rangeName = getRangeDescription(counterDataImage, rangeIndex)}; + rangeData.values.resize(metricNames.size()); + + // First set Counter data image with current range + NVPW_MetricsContext_SetCounterData_Params setCounterDataParams = { + NVPW_MetricsContext_SetCounterData_Params_STRUCT_SIZE, nullptr}; + + setCounterDataParams.pMetricsContext = + metricsContextCreateParams.pMetricsContext; + setCounterDataParams.pCounterDataImage = counterDataImage.data(); + setCounterDataParams.isolated = isolated; + setCounterDataParams.rangeIndex = rangeIndex; + + NVPW_CALL(NVPW_MetricsContext_SetCounterData(&setCounterDataParams)); + + + // Now we can evaluate GPU metrics + NVPW_MetricsContext_EvaluateToGpuValues_Params evalToGpuParams = { + NVPW_MetricsContext_EvaluateToGpuValues_Params_STRUCT_SIZE, nullptr}; + evalToGpuParams.pMetricsContext = + metricsContextCreateParams.pMetricsContext; + evalToGpuParams.numMetrics = metricNamePtrs.size(); + evalToGpuParams.ppMetricNames = metricNamePtrs.data(); + evalToGpuParams.pMetricValues = rangeData.values.data(); + + if (!NVPW_CALL(NVPW_MetricsContext_EvaluateToGpuValues(&evalToGpuParams))) { + LOG(WARNING) << "Failed to evaluate metris for range : " + << rangeData.rangeName; + continue; + } + + if (verbose) { + for (size_t i = 0; i < metricNames.size(); i++) { + LOG(INFO) << "rangeName: " << rangeData.rangeName + << "\tmetricName: " << metricNames[i] + << "\tgpuValue: " << rangeData.values[i]; + } + } + + result.rangeVals.emplace_back(std::move(rangeData)); + } + + return result; +} + +#else + +bool getProfilerConfigImage( + const std::string& /*chipName*/, + const std::vector& /*metricNames*/, + std::vector& /*configImage*/, + const uint8_t* /*counterAvailabilityImage*/) { + return false; +} + +bool getCounterDataPrefixImage( + const std::string& /*chipName*/, + const std::vector& /*metricNames*/, + std::vector& /*counterDataImagePrefix*/) { + return false; +} + +CuptiProfilerResult evalMetricValues( + const std::string& /*chipName*/, + const std::vector& /*counterDataImage*/, + const std::vector& /*metricNames*/, + bool /*verbose*/) { + return {}; +} + +#endif // cuda version > 10.00 and < 11.04 + +} // namespace nvperf +} // namespace KINETO_NAMESPACE diff --git a/tb_plugins/profiling/libkineto/src/CuptiNvPerfMetric.h b/tb_plugins/profiling/libkineto/src/CuptiNvPerfMetric.h new file mode 100644 index 000000000..d5dd1b1c1 --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/CuptiNvPerfMetric.h @@ -0,0 +1,71 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#pragma once + +#include +#include +#include + +// TODO(T90238193) +// @lint-ignore-every CLANGTIDY facebook-hte-RelativeInclude +#include "Logger.h" + +namespace KINETO_NAMESPACE { + +struct CuptiRangeMeasurement { + std::string rangeName; + std::vector values; +}; + +struct CuptiProfilerResult { + std::vector metricNames; + // rangeName, list values + std::vector rangeVals; +}; + +/* Utilities for CUPTI and NVIDIA PerfWorks Metric API + */ + +#define NVPW_CALL(call) \ + [&]() -> bool { \ + NVPA_Status _status_ = call; \ + if (_status_ != NVPA_STATUS_SUCCESS) { \ + LOG(WARNING) << fmt::format( \ + "function {} failed with error ({})", \ + #call, \ + (int)_status_); \ + return false; \ + } \ + return true; \ + }() + +// fixme - add a results string +// nvpperfGetResultString(_status_, &_errstr_); + +namespace nvperf { + +// Setup CUPTI profiler configuration blob and counter data image prefix +bool getProfilerConfigImage( + const std::string& chipName, + const std::vector& metricNames, + std::vector& configImage, + const uint8_t* counterAvailabilityImage = nullptr); + +// Setup CUPTI profiler configuration blob and counter data image prefix +bool getCounterDataPrefixImage( + const std::string& chipName, + const std::vector& metricNames, + std::vector& counterDataImagePrefix); + +/* NV Perf Metric Evaluation helpers + * - utilities to read binary data and obtain metrics for ranges + */ +CuptiProfilerResult evalMetricValues( + const std::string& chipName, + const std::vector& counterDataImage, + const std::vector& metricNames, + bool verbose = false); + + +} // namespace nvperf +} // namespace KINETO_NAMESPACE diff --git a/tb_plugins/profiling/libkineto/src/CuptiRangeProfilerApi.cpp b/tb_plugins/profiling/libkineto/src/CuptiRangeProfilerApi.cpp new file mode 100644 index 000000000..e5f18ed7b --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/CuptiRangeProfilerApi.cpp @@ -0,0 +1,751 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#include +#include +#ifdef HAS_CUPTI +#include +#include +#endif // HAS_CUPTI +#include +#include + +#ifdef HAS_CUPTI +#include "cupti_call.h" +#endif + +#include "time_since_epoch.h" +#include "Logger.h" +#include "Demangle.h" + +// TODO(T90238193) +// @lint-ignore-every CLANGTIDY facebook-hte-RelativeInclude +#include "CuptiCallbackApiMock.h" +#include "CuptiRangeProfilerApi.h" + +#if HAS_CUPTI_RANGE_PROFILER +#include +#include +#include "cupti_call.h" +#endif // HAS_CUPTI_RANGE_PROFILER + +namespace KINETO_NAMESPACE { + +#if HAS_CUPTI_RANGE_PROFILER +constexpr char kRootUserRangeName[] = "__profile__"; +constexpr int kCallbacksCountToFlush = 500; + +// Should we set Counter availability image ourselves? +// Disabled this right now as this call conflicts with DCGM +// It is not clear why it should conflict except it being a profiler API call +// TODO Revisit +constexpr bool kSetCounterAvail = false; + +// Shared state to track one Cupti Profiler API per Device +namespace { +// per device profiler maps +std::unordered_map profiler_map; +std::unordered_map enable_flag; +std::unordered_map disable_flag; + +std::mutex contextMutex_; +std::unordered_map ctx_to_dev; +std::set active_devices; +} + +// forward declarations +void __trackCudaCtx(CUcontext ctx, uint32_t device_id, CUpti_CallbackId cbid); +void __trackCudaKernelLaunch(CUcontext ctx, const char* kernelName); + +/// Helper functions + +// Available raw counters +std::vector getCounterAvailiability(CUcontext cuContext) { + std::vector counterAvailabilityImage; + CUpti_Profiler_GetCounterAvailability_Params getCounterAvailabilityParams = { + CUpti_Profiler_GetCounterAvailability_Params_STRUCT_SIZE, nullptr}; + getCounterAvailabilityParams.ctx = cuContext; + CUPTI_CALL( + cuptiProfilerGetCounterAvailability(&getCounterAvailabilityParams)); + + counterAvailabilityImage.clear(); + counterAvailabilityImage.resize( + getCounterAvailabilityParams.counterAvailabilityImageSize); + + getCounterAvailabilityParams.pCounterAvailabilityImage = + counterAvailabilityImage.data(); + CUPTI_CALL( + cuptiProfilerGetCounterAvailability(&getCounterAvailabilityParams)); + + return counterAvailabilityImage; +} + +std::string getChipName(int deviceId) { + // Get chip name for the cuda device + CUpti_Device_GetChipName_Params getChipNameParams = { + CUpti_Device_GetChipName_Params_STRUCT_SIZE, nullptr}; + + getChipNameParams.deviceIndex = deviceId; + CUPTI_CALL(cuptiDeviceGetChipName(&getChipNameParams)); + + return getChipNameParams.pChipName; +} + +inline uint32_t getDevID(CUcontext ctx) { + uint32_t device_id = UINT32_MAX; + CUPTI_CALL(cuptiGetDeviceId(ctx, &device_id)); + if (device_id == UINT32_MAX) { + LOG(ERROR) << "Could not determine dev id for = " << ctx; + } + return device_id; +} + +// We use CUPTI Callback functions in three ways : +// 1. Track cuda contexts and maintain a list of active GPUs to profile +// 2. Callbacks on kernel launches to track the name of automatic +// ranges that correspond to names of kernels +// 3. Lastly CUPTI profiler has to be enabled on the same thread executing +// the CUDA kernels. We use Callbacks to enable the profiler +// asynchronously from another thread. + +void disableKernelCallbacks(); + +void trackCudaCtx( + CUpti_CallbackDomain /*domain*/, + CUpti_CallbackId cbid, + const CUpti_CallbackData* cbInfo) { + auto *d = reinterpret_cast(cbInfo); + auto ctx = d->context; + uint32_t device_id = getDevID(ctx); + + if (device_id == UINT32_MAX) { + return; + } + + __trackCudaCtx(ctx, device_id, cbid); +} + +void __trackCudaCtx(CUcontext ctx, uint32_t device_id, CUpti_CallbackId cbid) { + std::lock_guard g(contextMutex_); + if (cbid == CUPTI_CBID_RESOURCE_CONTEXT_CREATED) { + VLOG(0) << "CUPTI Profiler observed CUDA Context created = " + << ctx << " device id = " << device_id; + active_devices.insert(device_id); + if constexpr (kSetCounterAvail) { + if (active_devices.size() == 1) { + CuptiRBProfilerSession::setCounterAvailabilityImage( + getCounterAvailiability(ctx)); + } + } + ctx_to_dev[ctx] = device_id; + + } else if (cbid == CUPTI_CBID_RESOURCE_CONTEXT_DESTROY_STARTING) { + VLOG(0) << "CUPTI Profiler observed CUDA Context destroyed = " + << ctx << " device id = " << device_id; + auto it = active_devices.find(device_id); + if (it != active_devices.end()) { + active_devices.erase(it); + ctx_to_dev.erase(ctx); + } + } +} + +void trackCudaKernelLaunch( + CUpti_CallbackDomain /*domain*/, + CUpti_CallbackId /*cbid*/, + const CUpti_CallbackData* cbInfo) { + VLOG(1) << " Trace : Callback name = " + << (cbInfo->symbolName ? cbInfo->symbolName: "") + << " context ptr = " << cbInfo->context; + auto ctx = cbInfo->context; + // should be in CUPTI_API_ENTER call site + if (cbInfo->callbackSite != CUPTI_API_ENTER) { + return; + } + __trackCudaKernelLaunch(ctx, cbInfo->symbolName); +} + +void __trackCudaKernelLaunch( + CUcontext ctx, + const char* kernelName) { + VLOG(0) << " Tracking kernel name = " << (kernelName ? kernelName : "") + << " context ptr = " << ctx; + + uint32_t device_id = 0; + auto it = ctx_to_dev.find(ctx); + if (it == ctx_to_dev.end()) { + // Warning here could be too noisy + VLOG(0) << " Could not find corresponding device to ctx = " << ctx; + return; + } else { + device_id = it->second; + } + + auto pit = profiler_map.find(device_id); + if (pit == profiler_map.end() || pit->second == nullptr) { + return; + } + auto profiler = pit->second; + + if (enable_flag[device_id]) { + LOG(INFO) << "Callback handler is enabling cupti profiler"; + profiler->startAndEnable(); + enable_flag[device_id] = false; + + } else if (disable_flag[device_id]) { + LOG(INFO) << "Callback handler is disabling cupti profiler"; + profiler->disableAndStop(); + return; + } + + if (profiler->curRange_ == CUPTI_AutoRange) { + profiler->logKernelName(kernelName ? kernelName : "__missing__"); + } + + /* TODO add per kernel time logging + if (measure_per_kernel) { + profiler->kernelStartTs_.push_back( + std::chrono::high_resolution_clock::now()); + } + */ + + // periodically flush profiler data from GPU + if (profiler->numCallbacks_ % kCallbacksCountToFlush == 0) { + profiler->flushCounterData(); + } + profiler->numCallbacks_++; +} + +void enableKernelCallbacks() { + auto& cbapi = CuptiCallbackApi::singleton(); + bool status = cbapi.enableCallback( + CUPTI_CB_DOMAIN_RUNTIME_API, + CUPTI_RUNTIME_TRACE_CBID_cudaLaunchKernel_v7000); + if (!status) { + LOG(WARNING) << "CUPTI Range Profiler unable to " + << "enable cuda kernel launch callback"; + return; + } + LOG(INFO) << "CUPTI Profiler kernel callbacks enabled"; +} + +void disableKernelCallbacks() { + auto& cbapi = CuptiCallbackApi::singleton(); + bool status = cbapi.disableCallback( + CUPTI_CB_DOMAIN_RUNTIME_API, + CUPTI_RUNTIME_TRACE_CBID_cudaLaunchKernel_v7000); + if (!status) { + LOG(WARNING) << "CUPTI Range Profiler unable to " + << "disable cuda kernel launch callback"; + return; + } + LOG(INFO) << "CUPTI Profiler kernel callbacks disabled"; +} + +// static +std::set CuptiRBProfilerSession::getActiveDevices() { + std::lock_guard g(contextMutex_); + return active_devices; +} + +// static +void CuptiRBProfilerSession::initCupti() { + CUpti_Profiler_Initialize_Params profilerInitializeParams = { + CUpti_Profiler_Initialize_Params_STRUCT_SIZE, nullptr}; + CUPTI_CALL(cuptiProfilerInitialize(&profilerInitializeParams)); +} + +// static +void CuptiRBProfilerSession::deInitCupti() { + CUpti_Profiler_DeInitialize_Params profilerDeInitializeParams = { + CUpti_Profiler_DeInitialize_Params_STRUCT_SIZE, nullptr}; + CUPTI_CALL(cuptiProfilerDeInitialize(&profilerDeInitializeParams)); +} + +// static +void CuptiRBProfilerSession::staticInit() { + CuptiRBProfilerSession::initCupti(); + + // Register CUPTI callbacks + auto& cbapi = CuptiCallbackApi::singleton(); + CUpti_CallbackDomain domain = CUPTI_CB_DOMAIN_RESOURCE; + bool status = cbapi.registerCallback( + domain, CuptiCallbackApi::RESOURCE_CONTEXT_CREATED, trackCudaCtx); + status = status && cbapi.registerCallback( + domain, CuptiCallbackApi::RESOURCE_CONTEXT_DESTROYED, trackCudaCtx); + status = status && cbapi.enableCallback( + domain, CUPTI_CBID_RESOURCE_CONTEXT_CREATED); + status = status && cbapi.enableCallback( + domain, CUPTI_CBID_RESOURCE_CONTEXT_DESTROY_STARTING); + + if (!status) { + LOG(WARNING) << "CUPTI Range Profiler unable to attach cuda context " + << "create and destroy callbacks"; + CUPTI_CALL(cbapi.getCuptiStatus()); + return; + } + + domain = CUPTI_CB_DOMAIN_RUNTIME_API; + status = cbapi.registerCallback( + domain, CuptiCallbackApi::CUDA_LAUNCH_KERNEL, trackCudaKernelLaunch); + + if (!status) { + LOG(WARNING) << "CUPTI Range Profiler unable to attach cuda kernel " + << "launch callback"; + return; + } +} + +// static +std::vector& CuptiRBProfilerSession::counterAvailabilityImage() { + static std::vector counterAvailabilityImage_; + return counterAvailabilityImage_; +} + + +// Setup the profiler sessions +CuptiRBProfilerSession::CuptiRBProfilerSession( + const std::vector& metricNames, + int deviceId, + int maxRanges, + int numNestingLevels, + CUcontext cuContext) + : metricNames_(metricNames), + chipName_(getChipName(deviceId)), + deviceId_(deviceId), + maxRanges_(maxRanges), + numNestingLevels_(numNestingLevels), + cuContext_(cuContext) { + CuptiRBProfilerSession::initCupti(); + + LOG(INFO) << "Initializing CUPTI profiler session : device = " << deviceId + << " chip = " << chipName_; + /* Generate configuration for metrics, this can also be done offline*/ + NVPW_InitializeHost_Params initializeHostParams = { + NVPW_InitializeHost_Params_STRUCT_SIZE, nullptr}; + NVPW_CALL(NVPW_InitializeHost(&initializeHostParams)); + + if (metricNames.size()) { + if (!nvperf::getProfilerConfigImage( + chipName_, + metricNames, + configImage, + CuptiRBProfilerSession::counterAvailabilityImage().data())) { + LOG(ERROR) << "Failed to create configImage or counterDataImagePrefix"; + return; + } + if (!nvperf::getCounterDataPrefixImage( + chipName_, + metricNames, + counterDataImagePrefix)) { + LOG(ERROR) << "Failed to create counterDataImagePrefix"; + return; + } + } else { + LOG(ERROR) << "No metrics provided to profile"; + return; + } + + if (!createCounterDataImage()) { + LOG(ERROR) << "Failed to create counterDataImage"; + return; + } + + LOG(INFO) << "Size of structs\n" + << " config image size = " << configImage.size() << " B" + << " counter data image prefix = " + << counterDataImagePrefix.size() << " B" + << " counter data image size = " << counterDataImage.size() / 1024 + << " KB" + << " counter sb image size = " + << counterDataScratchBuffer.size() << " B"; + + beginPassParams_ = {CUpti_Profiler_BeginPass_Params_STRUCT_SIZE, nullptr}; + endPassParams_ = {CUpti_Profiler_EndPass_Params_STRUCT_SIZE, nullptr}; + + initSuccess_ = true; + profiler_map[deviceId] = this; +} + +// used in unittests only +CuptiRBProfilerSession::CuptiRBProfilerSession(int deviceId, CUcontext ctx) + : deviceId_(deviceId), cuContext_(ctx) { + initSuccess_ = true; + profiler_map[deviceId] = this; +} + +void CuptiRBProfilerSession::startInternal( + CUpti_ProfilerRange profilerRange, + CUpti_ProfilerReplayMode profilerReplayMode) { + LOG(INFO) << "Starting profiler session: profiler range = " + << ((profilerRange == CUPTI_AutoRange) ? "autorange" : "userrange") + << " replay mode = " + << ((profilerReplayMode == CUPTI_KernelReplay) ? "kernel" : "user"); + if (!initSuccess_) { + LOG(WARNING) << __func__ << "() bailing out since initialization failed"; + return; + } + + if (cuContext_ == nullptr) { + for (const auto& it : ctx_to_dev) { + if (it.second == deviceId_) { + cuContext_ = it.first; + break; + } + } + LOG(INFO) << " Cupti Profiler using CUDA context = " << cuContext_; + } + + profilerStartTs_ = std::chrono::high_resolution_clock::now(); + curRange_ = profilerRange; + curReplay_ = profilerReplayMode; + + CUpti_Profiler_BeginSession_Params beginSessionParams = { + CUpti_Profiler_BeginSession_Params_STRUCT_SIZE, nullptr}; + + beginSessionParams.ctx = cuContext_; + beginSessionParams.counterDataImageSize = counterDataImage.size(); + beginSessionParams.pCounterDataImage = counterDataImage.data(); + beginSessionParams.counterDataScratchBufferSize = + counterDataScratchBuffer.size(); + beginSessionParams.pCounterDataScratchBuffer = counterDataScratchBuffer.data(); + beginSessionParams.range = profilerRange; + beginSessionParams.replayMode = profilerReplayMode; + beginSessionParams.maxRangesPerPass = maxRanges_; + beginSessionParams.maxLaunchesPerPass = maxRanges_; + + auto status = CUPTI_CALL(cuptiProfilerBeginSession(&beginSessionParams)); + if (status != CUPTI_SUCCESS) { + LOG(WARNING) << "Failed to start CUPTI profiler"; + initSuccess_ = false; + return; + } + + // Set counter configuration + CUpti_Profiler_SetConfig_Params setConfigParams = { + CUpti_Profiler_SetConfig_Params_STRUCT_SIZE, nullptr}; + + setConfigParams.ctx = cuContext_; + setConfigParams.pConfig = configImage.data(); + setConfigParams.configSize = configImage.size(); + setConfigParams.passIndex = 0; + setConfigParams.minNestingLevel = 1; + setConfigParams.numNestingLevels = numNestingLevels_; + status = CUPTI_CALL(cuptiProfilerSetConfig(&setConfigParams)); + + if (status != CUPTI_SUCCESS) { + LOG(WARNING) << "Failed to configure CUPTI profiler"; + initSuccess_ = false; + return; + } + profilerInitDoneTs_ = std::chrono::high_resolution_clock::now(); + + if (curRange_ == CUPTI_AutoRange) { + enableKernelCallbacks(); + } + profilingActive_ = true; +} + +void CuptiRBProfilerSession::stop() { + if (!initSuccess_) { + LOG(WARNING) << __func__ << "() bailing out since initialization failed"; + return; + } + LOG(INFO) << "Stop profiler session on device = " << deviceId_; + + CUpti_Profiler_UnsetConfig_Params unsetConfigParams = { + CUpti_Profiler_UnsetConfig_Params_STRUCT_SIZE, nullptr}; + CUPTI_CALL(cuptiProfilerUnsetConfig(&unsetConfigParams)); + + CUpti_Profiler_EndSession_Params endSessionParams = { + CUpti_Profiler_EndSession_Params_STRUCT_SIZE, nullptr}; + CUPTI_CALL(cuptiProfilerEndSession(&endSessionParams)); + + disableKernelCallbacks(); + + profilerStopTs_ = std::chrono::high_resolution_clock::now(); + profilingActive_ = false; +} + +void CuptiRBProfilerSession::beginPass() { + if (!initSuccess_) { + LOG(WARNING) << __func__ << "() bailing out since initialization failed"; + return; + } + CUPTI_CALL(cuptiProfilerBeginPass(&beginPassParams_)); +} + +bool CuptiRBProfilerSession::endPass() { + if (!initSuccess_) { + LOG(WARNING) << __func__ << "() bailing out since initialization failed"; + return true; + } + CUPTI_CALL(cuptiProfilerEndPass(&endPassParams_)); + return endPassParams_.allPassesSubmitted; +} + +void CuptiRBProfilerSession::flushCounterData() { + LOG(INFO) << "Flushing counter data on device = " << deviceId_; + CUpti_Profiler_FlushCounterData_Params flushCounterDataParams = { + CUpti_Profiler_FlushCounterData_Params_STRUCT_SIZE, nullptr}; + CUPTI_CALL(cuptiProfilerFlushCounterData(&flushCounterDataParams)); +} + +/// Enable and disable the profiler +void CuptiRBProfilerSession::enable() { + if (!initSuccess_) { + LOG(WARNING) << __func__ << "() bailing out since initialization failed"; + return; + } + CUpti_Profiler_EnableProfiling_Params enableProfilingParams = { + CUpti_Profiler_EnableProfiling_Params_STRUCT_SIZE, nullptr}; + CUPTI_CALL(cuptiProfilerEnableProfiling(&enableProfilingParams)); +} + +void CuptiRBProfilerSession::disable() { + if (!initSuccess_) { + LOG(WARNING) << __func__ << "() bailing out since initialization failed"; + return; + } + CUpti_Profiler_DisableProfiling_Params disableProfilingParams = { + CUpti_Profiler_DisableProfiling_Params_STRUCT_SIZE, nullptr}; + CUPTI_CALL(cuptiProfilerDisableProfiling(&disableProfilingParams)); +} + +/// User range based profiling +void CuptiRBProfilerSession::pushRange(const std::string& rangeName) { + LOG(INFO) << " CUPTI pushrange ( " << rangeName << " )"; + CUpti_Profiler_PushRange_Params pushRangeParams = { + CUpti_Profiler_PushRange_Params_STRUCT_SIZE, nullptr}; + pushRangeParams.pRangeName = rangeName.c_str(); + CUPTI_CALL(cuptiProfilerPushRange(&pushRangeParams)); +} + +void CuptiRBProfilerSession::popRange() { + LOG(INFO) << " CUPTI pop range"; + CUpti_Profiler_PopRange_Params popRangeParams = { + CUpti_Profiler_PopRange_Params_STRUCT_SIZE, nullptr}; + CUPTI_CALL(cuptiProfilerPopRange(&popRangeParams)); +} + +void CuptiRBProfilerSession::startAndEnable() { + startInternal(curRange_, curReplay_); + if (curReplay_ == CUPTI_UserReplay) { + beginPass(); + } + enable(); + if (curRange_ == CUPTI_UserRange) { + pushRange(kRootUserRangeName); + } + enable_flag[deviceId_] = false; +} + +void CuptiRBProfilerSession::disableAndStop() { + if (curRange_ == CUPTI_UserRange) { + popRange(); + } + disable(); + if (curReplay_ == CUPTI_UserReplay) { + endPass(); + flushCounterData(); + } + stop(); + disable_flag[deviceId_] = false; +} + +void CuptiRBProfilerSession::asyncStartAndEnable( + CUpti_ProfilerRange profilerRange, + CUpti_ProfilerReplayMode profilerReplayMode) { + LOG(INFO) << "Starting CUPTI profiler asynchronously on device = " + << deviceId_ << " profiler range = " + << ((profilerRange == CUPTI_AutoRange) ? "autorange" : "userrange") + << " replay mode = " + << ((profilerReplayMode == CUPTI_KernelReplay) ? "kernel" : "user"); + curReplay_ = profilerReplayMode; + curRange_ = profilerRange; + enable_flag[deviceId_] = true; + enableKernelCallbacks(); +} + +void CuptiRBProfilerSession::asyncDisableAndStop() { + LOG(INFO) << "Stopping CUPTI profiler asynchronously on device = " + << deviceId_ << " cu context = " << cuContext_; + disable_flag[deviceId_] = true; +} + + +CuptiProfilerResult CuptiRBProfilerSession::evaluateMetrics( + bool verbose) { + if (!initSuccess_) { + LOG(WARNING) << "Profiling failed, no results to return"; + return {}; + } + if (profilingActive_) { + disableAndStop(); + } + + LOG(INFO) << "Total kernels logged = " << kernelNames_.size(); + if (verbose) { + for (const auto& kernel : kernelNames_) { + std::cout << demangle(kernel) << std::endl; + } + LOG(INFO) << "Profiler Range data : "; + } + + auto results = nvperf::evalMetricValues( + chipName_, counterDataImage, metricNames_, verbose /*verbose*/); + + // profiler end-end duration + auto duration_ms = std::chrono::duration_cast( + profilerStopTs_ - profilerStartTs_); + + auto init_dur_ms = std::chrono::duration_cast( + profilerInitDoneTs_ - profilerStartTs_); + LOG(INFO) << "Total profiler time = " << duration_ms.count() << " ms"; + LOG(INFO) << "Total profiler init time = " << init_dur_ms.count() << " ms"; + + return results; +} + +std::unique_ptr CuptiRBProfilerSession::getProfilerTraceSpan() { + return std::make_unique( + timeSinceEpoch(profilerStartTs_), + timeSinceEpoch(profilerStopTs_), + "__cupti_profiler__" + ); +} + +void CuptiRBProfilerSession::saveCounterData( + const std::string& /*CounterDataFileName*/, + const std::string& /*CounterDataSBFileName*/) { + /* TBD write binary files for counter data and counter scratch buffer */ +} + +/// Setup counter data +bool CuptiRBProfilerSession::createCounterDataImage() { + CUpti_Profiler_CounterDataImageOptions counterDataImageOptions; + counterDataImageOptions.pCounterDataPrefix = counterDataImagePrefix.data(); + counterDataImageOptions.counterDataPrefixSize = counterDataImagePrefix.size(); + counterDataImageOptions.maxNumRanges = maxRanges_; + counterDataImageOptions.maxNumRangeTreeNodes = maxRanges_; + counterDataImageOptions.maxRangeNameLength = 64; + + // Calculate size of counter data image + CUpti_Profiler_CounterDataImage_CalculateSize_Params calculateSizeParams = { + CUpti_Profiler_CounterDataImage_CalculateSize_Params_STRUCT_SIZE, nullptr}; + calculateSizeParams.pOptions = &counterDataImageOptions; + calculateSizeParams.sizeofCounterDataImageOptions = + CUpti_Profiler_CounterDataImageOptions_STRUCT_SIZE; + + CUPTI_CALL( + cuptiProfilerCounterDataImageCalculateSize(&calculateSizeParams)); + counterDataImage.resize(calculateSizeParams.counterDataImageSize); + + // Initialize counter data image + CUpti_Profiler_CounterDataImage_Initialize_Params initializeParams = { + CUpti_Profiler_CounterDataImage_Initialize_Params_STRUCT_SIZE, nullptr}; + initializeParams.sizeofCounterDataImageOptions = + CUpti_Profiler_CounterDataImageOptions_STRUCT_SIZE; + initializeParams.pOptions = &counterDataImageOptions; + initializeParams.counterDataImageSize = + calculateSizeParams.counterDataImageSize; + initializeParams.pCounterDataImage = counterDataImage.data(); + CUPTI_CALL(cuptiProfilerCounterDataImageInitialize(&initializeParams)); + + // Calculate counter Scratch Buffer size + CUpti_Profiler_CounterDataImage_CalculateScratchBufferSize_Params + scratchBufferSizeParams = { + CUpti_Profiler_CounterDataImage_CalculateScratchBufferSize_Params_STRUCT_SIZE, nullptr}; + + scratchBufferSizeParams.counterDataImageSize = + calculateSizeParams.counterDataImageSize; + scratchBufferSizeParams.pCounterDataImage = + initializeParams.pCounterDataImage; + CUPTI_CALL(cuptiProfilerCounterDataImageCalculateScratchBufferSize( + &scratchBufferSizeParams)); + + counterDataScratchBuffer.resize( + scratchBufferSizeParams.counterDataScratchBufferSize); + + // Initialize scratch buffer + CUpti_Profiler_CounterDataImage_InitializeScratchBuffer_Params + initScratchBufferParams = { + CUpti_Profiler_CounterDataImage_InitializeScratchBuffer_Params_STRUCT_SIZE, nullptr}; + + initScratchBufferParams.counterDataImageSize = + calculateSizeParams.counterDataImageSize; + + initScratchBufferParams.pCounterDataImage = + initializeParams.pCounterDataImage; + initScratchBufferParams.counterDataScratchBufferSize = + scratchBufferSizeParams.counterDataScratchBufferSize; + initScratchBufferParams.pCounterDataScratchBuffer = + counterDataScratchBuffer.data(); + + CUPTI_CALL(cuptiProfilerCounterDataImageInitializeScratchBuffer( + &initScratchBufferParams)); + + return true; +} + +#elif defined(HAS_CUPTI) + +// Create empty stubs for the API when CUPTI is not present. +CuptiRBProfilerSession::CuptiRBProfilerSession( + const std::vector& metricNames, + int deviceId, + int maxRanges, + int numNestingLevels, + CUcontext cuContext) + : metricNames_(metricNames), + deviceId_(deviceId), + maxRanges_(maxRanges), + numNestingLevels_(numNestingLevels), + cuContext_(cuContext) {} +void CuptiRBProfilerSession::stop() {} +void CuptiRBProfilerSession::enable() {} +void CuptiRBProfilerSession::disable() {} +void CuptiRBProfilerSession::beginPass() {} +bool CuptiRBProfilerSession::endPass() { return true; } +void CuptiRBProfilerSession::flushCounterData() {} +void CuptiRBProfilerSession::pushRange(const std::string& /*rangeName*/) {} +void CuptiRBProfilerSession::popRange() {} +void CuptiRBProfilerSession::asyncStartAndEnable( + CUpti_ProfilerRange /*profilerRange*/, + CUpti_ProfilerReplayMode /*profilerReplayMode*/) {} +void CuptiRBProfilerSession::asyncDisableAndStop() {} +CuptiProfilerResult CuptiRBProfilerSession::evaluateMetrics(bool verbose) { + static CuptiProfilerResult res; + return res; +}; +void CuptiRBProfilerSession::saveCounterData( + const std::string& /*CounterDataFileName*/, + const std::string& /*CounterDataSBFileName*/) {} +void CuptiRBProfilerSession::initCupti() {} +void CuptiRBProfilerSession::deInitCupti() {} +void CuptiRBProfilerSession::staticInit() {} +bool CuptiRBProfilerSession::createCounterDataImage() { return true; } +void CuptiRBProfilerSession::startInternal( + CUpti_ProfilerRange /*profilerRange*/, + CUpti_ProfilerReplayMode /*profilerReplayMode*/) {} +std::vector& CuptiRBProfilerSession::counterAvailabilityImage() { + static std::vector _vec; + return _vec; +} +#endif // HAS_CUPTI_RANGE_PROFILER + +namespace testing { + +void trackCudaCtx(CUcontext ctx, uint32_t device_id, CUpti_CallbackId cbid) { +#if HAS_CUPTI_RANGE_PROFILER + __trackCudaCtx(ctx, device_id, cbid); +#endif // HAS_CUPTI_RANGE_PROFILER +} + +void trackCudaKernelLaunch(CUcontext ctx, const char* kernelName) { +#if HAS_CUPTI_RANGE_PROFILER + __trackCudaKernelLaunch(ctx, kernelName); +#endif // HAS_CUPTI_RANGE_PROFILER +} + +} // namespace testing +} // namespace KINETO_NAMESPACE diff --git a/tb_plugins/profiling/libkineto/src/CuptiRangeProfilerApi.h b/tb_plugins/profiling/libkineto/src/CuptiRangeProfilerApi.h new file mode 100644 index 000000000..98a0b3ea5 --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/CuptiRangeProfilerApi.h @@ -0,0 +1,220 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#pragma once + +#ifdef HAS_CUPTI +#include +#include +// Using CUDA 11 and above due to usage of API: cuptiProfilerGetCounterAvailability. +#if defined(CUDART_VERSION) && CUDART_VERSION >= 10000 && CUDART_VERSION < 11040 && CUDA_VERSION >= 11000 +#define HAS_CUPTI_RANGE_PROFILER 1 +#endif // CUDART_VERSION > 10.00 and < 11.04 && CUDA_VERSION >= 11.00 +#endif // HAS_CUPTI + +#if HAS_CUPTI_RANGE_PROFILER +#include +#include +#include +#else +using CUpti_ProfilerRange = enum +{ + CUPTI_AutoRange, + CUPTI_UserRange, +}; + +using CUpti_ProfilerReplayMode = enum +{ + CUPTI_KernelReplay, + CUPTI_UserReplay, +}; +#endif // HAS_CUPTI_RANGE_PROFILER + +#include +#include +#include +#include +#include + +// TODO(T90238193) +// @lint-ignore-every CLANGTIDY facebook-hte-RelativeInclude +#include "TraceSpan.h" +#include "CuptiCallbackApi.h" +#include "CuptiNvPerfMetric.h" + +/* Cupti Range based profiler session + * See : https://docs.nvidia.com/cupti/Cupti/r_main.html#r_profiler + */ + +namespace KINETO_NAMESPACE { + +class CuptiRBProfilerSession { + public: + // Initialize and configure CUPTI Profiler counters. + // - Metric names must be provided as string vector. + // - Supported values by CUPTI can be found at - + // https://docs.nvidia.com/cupti/Cupti/r_main.html#r_host_metrics_api + explicit CuptiRBProfilerSession( + const std::vector& metricNames, + int deviceId, + int maxRanges, + int numNestingLevels = 1, + CUcontext cuContext = 0); + + virtual ~CuptiRBProfilerSession() = default; + + // Start profiling session + // This function has to be called from the CPU thread running + // the CUDA context. If this is not the case asyncStartAndEnable() + // can be used + void start( + CUpti_ProfilerRange profilerRange = CUPTI_AutoRange, + CUpti_ProfilerReplayMode profilerReplayMode = CUPTI_KernelReplay) { + startInternal(profilerRange, profilerReplayMode); + } + + // Stop profiling session + virtual void stop(); + + virtual void enable(); + virtual void disable(); + + // Profiler passes + // GPU hardware has limited performance monitoring resources + // the CUPTI profiler may need to run multiple passes to collect + // data for a given range + // If we use kernel replay model the kernels are automatically replayed + // else, you can use the beginPass() and endPass() functions below + // for user to manage the replays + + // starts a profiler pass with given kernels in between + virtual void beginPass(); + + // end a profiler pass with given kernels in between + // returns true if no more passes are required + virtual bool endPass(); + + // flushes the counter data - required if you use user replay + virtual void flushCounterData(); + + // Each pass can contain multiple of ranges + // metrics configured in a pass are collected per each range-stack. + virtual void pushRange(const std::string& rangeName); + virtual void popRange(); + + // utilities for common operations + void startAndEnable(); + void disableAndStop(); + + // Async APIs : these will can be called from another thread + // outside the CUDA context being profiled + void asyncStartAndEnable( + CUpti_ProfilerRange profilerRange = CUPTI_AutoRange, + CUpti_ProfilerReplayMode profilerReplayMode = CUPTI_KernelReplay); + void asyncDisableAndStop(); + + void printMetrics() { + evaluateMetrics(true); + } + + std::unique_ptr getProfilerTraceSpan(); + + virtual CuptiProfilerResult evaluateMetrics(bool verbose = false); + + void saveCounterData( + const std::string& CounterDataFileName, + const std::string& CounterDataSBFileName); + + // This is not thread safe so please only call after + // profiling has stopped + const std::vector& getKernelNames() const { + return kernelNames_; + } + + int deviceId() const { + return deviceId_; + } + + bool profilingActive() const { + return profilingActive_; + } + + static std::set getActiveDevices(); + + static void initCupti(); + + static void deInitCupti(); + + static void staticInit(); + + static void setCounterAvailabilityImage(std::vector img) { + counterAvailabilityImage() = img; + } + protected: + CuptiRBProfilerSession(int deviceId, CUcontext ctx); + + virtual void startInternal( + CUpti_ProfilerRange profilerRange, + CUpti_ProfilerReplayMode profilerReplayMode); + + CUpti_ProfilerRange curRange_ = CUPTI_AutoRange; + CUpti_ProfilerReplayMode curReplay_ = CUPTI_KernelReplay; + + private: + + bool createCounterDataImage(); + + + // log kernel name that used with callbacks + void logKernelName(const char* kernel) { + std::lock_guard lg(kernelNamesMutex_); + kernelNames_.emplace_back(kernel); + } + + std::vector metricNames_; + std::string chipName_; + + uint32_t deviceId_ = 0; + int maxRanges_; + int numNestingLevels_; + CUcontext cuContext_; + + + // data buffers for configuration and counter data collection + std::vector counterDataImagePrefix; + std::vector configImage; + std::vector counterDataImage; + std::vector counterDataScratchBuffer; + + std::chrono::time_point profilerStartTs_; + std::chrono::time_point + profilerInitDoneTs_; + std::chrono::time_point profilerStopTs_; + + std::mutex kernelNamesMutex_; + // raw kernel names (not demangled) + std::vector kernelNames_; + + uint32_t numCallbacks_ = 0; + + static std::vector& counterAvailabilityImage(); + +#if HAS_CUPTI_RANGE_PROFILER + CUpti_Profiler_BeginPass_Params beginPassParams_; + CUpti_Profiler_EndPass_Params endPassParams_; +#endif + + bool initSuccess_ = false; + bool profilingActive_ = false; + + friend void __trackCudaKernelLaunch(CUcontext ctx, const char* kernelName); +}; + +// called directly only in unit tests +namespace testing { + +void trackCudaCtx(CUcontext ctx, uint32_t device_id, CUpti_CallbackId cbid); +void trackCudaKernelLaunch(CUcontext ctx, const char* kernelName); + +} // namespace testing + +} // namespace KINETO_NAMESPACE diff --git a/tb_plugins/profiling/libkineto/src/CuptiRangeProfilerConfig.cpp b/tb_plugins/profiling/libkineto/src/CuptiRangeProfilerConfig.cpp new file mode 100644 index 000000000..04b1ad0cb --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/CuptiRangeProfilerConfig.cpp @@ -0,0 +1,68 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#include +#include + +#include +#include + +#include +#include + +using namespace std::chrono; + +namespace KINETO_NAMESPACE { + +// number of ranges affect the size of counter data binary used by +// the CUPTI Profiler. these defaults can be tuned +constexpr int KMaxAutoRanges = 1500; // supports 1500 kernels +constexpr int KMaxUserRanges = 10; // enable upto 10 sub regions marked by user + +constexpr char kCuptiProfilerMetricsKey[] = "CUPTI_PROFILER_METRICS"; +constexpr char kCuptiProfilerPerKernelKey[] = "CUPTI_PROFILER_ENABLE_PER_KERNEL"; +constexpr char kCuptiProfilerMaxRangesKey[] = "CUPTI_PROFILER_MAX_RANGES"; + +CuptiRangeProfilerConfig::CuptiRangeProfilerConfig(Config& cfg) + : parent_(&cfg), + cuptiProfilerPerKernel_(false), + cuptiProfilerMaxRanges_(0) {} + +bool CuptiRangeProfilerConfig::handleOption(const std::string& name, std::string& val) { + VLOG(0) << " handling : " << name << " = " << val; + // Cupti Range based Profiler configuration + if (!name.compare(kCuptiProfilerMetricsKey)) { + activitiesCuptiMetrics_ = splitAndTrim(val, ','); + } else if (!name.compare(kCuptiProfilerPerKernelKey)) { + cuptiProfilerPerKernel_ = toBool(val); + } else if (!name.compare(kCuptiProfilerMaxRangesKey)) { + cuptiProfilerMaxRanges_ = toInt64(val); + } else { + return false; + } + return true; +} + +void CuptiRangeProfilerConfig::setDefaults() { + if (activitiesCuptiMetrics_.size() > 0 && cuptiProfilerMaxRanges_ == 0) { + cuptiProfilerMaxRanges_ = + cuptiProfilerPerKernel_ ? KMaxAutoRanges : KMaxUserRanges; + } +} + +void CuptiRangeProfilerConfig::printActivityProfilerConfig(std::ostream& s) const { + if (activitiesCuptiMetrics_.size() > 0) { + s << "Cupti Profiler metrics : " + << fmt::format("{}", fmt::join(activitiesCuptiMetrics_, ", ")) << std::endl; + s << "Cupti Profiler measure per kernel : " + << cuptiProfilerPerKernel_ << std::endl; + s << "Cupti Profiler max ranges : " << cuptiProfilerMaxRanges_ << std::endl; + } +} + +void CuptiRangeProfilerConfig::registerFactory() { + Config::addConfigFactory( + kCuptiProfilerConfigName, + [](Config& cfg) { return new CuptiRangeProfilerConfig(cfg); }); +} + +} // namespace KINETO_NAMESPACE diff --git a/tb_plugins/profiling/libkineto/src/CuptiRangeProfilerConfig.h b/tb_plugins/profiling/libkineto/src/CuptiRangeProfilerConfig.h new file mode 100644 index 000000000..549b8a4e8 --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/CuptiRangeProfilerConfig.h @@ -0,0 +1,86 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#pragma once + +#include "Config.h" + +#include +#include +#include +#include + +namespace KINETO_NAMESPACE { + +constexpr char kCuptiProfilerConfigName[] = "cupti_rb_profiler"; + +class CuptiRangeProfilerConfig : public AbstractConfig { + public: + bool handleOption(const std::string& name, std::string& val) override; + + void validate( + const std::chrono::time_point& + fallbackProfileStartTime) override {} + + static CuptiRangeProfilerConfig& get(const Config& cfg) { + return dynamic_cast(cfg.feature( + kCuptiProfilerConfigName)); + } + + Config& parent() const { + return *parent_; + } + + std::vector activitiesCuptiMetrics() const { + return activitiesCuptiMetrics_; + } + + bool cuptiProfilerPerKernel() const { + return cuptiProfilerPerKernel_; + } + + int64_t cuptiProfilerMaxRanges() const { + return cuptiProfilerMaxRanges_; + } + + void setSignalDefaults() override { + setDefaults(); + } + + void setClientDefaults() override { + setDefaults(); + } + + void printActivityProfilerConfig(std::ostream& s) const override; + + static void registerFactory(); + protected: + AbstractConfig* cloneDerived(AbstractConfig& parent) const override { + CuptiRangeProfilerConfig* clone = new CuptiRangeProfilerConfig(*this); + clone->parent_ = dynamic_cast(&parent); + return clone; + } + + private: + CuptiRangeProfilerConfig() = delete; + explicit CuptiRangeProfilerConfig(Config& parent); + explicit CuptiRangeProfilerConfig( + const CuptiRangeProfilerConfig& other) = default; + + // some defaults will depend on other configuration + void setDefaults(); + + // Associated Config object + Config* parent_; + + // Counter metrics exposed via CUPTI Profiler API + std::vector activitiesCuptiMetrics_; + + // Collect profiler metrics per kernel - autorange made + bool cuptiProfilerPerKernel_{false}; + + // max number of ranges to configure the profiler for. + // this has to be set before hand to reserve space for the output + int64_t cuptiProfilerMaxRanges_ = 0; +}; + +} // namespace KINETO_NAMESPACE diff --git a/tb_plugins/profiling/libkineto/src/DaemonConfigLoader.h b/tb_plugins/profiling/libkineto/src/DaemonConfigLoader.h new file mode 100644 index 000000000..9b0ed9286 --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/DaemonConfigLoader.h @@ -0,0 +1,27 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#pragma once + +#include +#include + +namespace KINETO_NAMESPACE { + +class DaemonConfigLoader { + public: + virtual ~DaemonConfigLoader() {} + + // Return the base config from the daemon + virtual std::string readBaseConfig() = 0; + + // Return a configuration string from the daemon, if one has been posted. + virtual std::string readOnDemandConfig(bool events, bool activities) = 0; + + // Returns the number of tracked contexts for this device. The daemon has a + // global view. If an unexpedted error occurs, return -1. + virtual int gpuContextCount(uint32_t device) = 0; + + virtual void setCommunicationFabric(bool enabled) = 0; +}; + +} // namespace KINETO_NAMESPACE diff --git a/tb_plugins/profiling/libkineto/src/Demangle.cpp b/tb_plugins/profiling/libkineto/src/Demangle.cpp new file mode 100644 index 000000000..f84f0b8ec --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/Demangle.cpp @@ -0,0 +1,49 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#include "Demangle.h" + +#ifndef _MSC_VER +#include +#endif +#include +#include + +namespace KINETO_NAMESPACE { + +static constexpr int kMaxSymbolSize = 1024; + +std::string demangle(const char* name) { +#ifndef _MSC_VER + if (!name) { + return ""; + } + + if (strlen(name) > kMaxSymbolSize) { + return name; + } + + int status; + size_t len = 0; + char* demangled = abi::__cxa_demangle(name, nullptr, &len, &status); + if (status != 0) { + return name; + } + std::string res(demangled); + // The returned buffer must be freed! + free(demangled); + return res; +#else + // TODO: demangling on Windows + if (!name) { + return ""; + } else { + return name; + } +#endif +} + +std::string demangle(const std::string& name) { + return demangle(name.c_str()); +} + +} // namespace KINETO_NAMESPACE diff --git a/tb_plugins/profiling/libkineto/src/Demangle.h b/tb_plugins/profiling/libkineto/src/Demangle.h new file mode 100644 index 000000000..6dcf0776f --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/Demangle.h @@ -0,0 +1,12 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#pragma once + +#include + +namespace KINETO_NAMESPACE { + +std::string demangle(const char* name); +std::string demangle(const std::string& name); + +} // namespace KINETO_NAMESPACE diff --git a/tb_plugins/profiling/libkineto/src/EventProfiler.cpp b/tb_plugins/profiling/libkineto/src/EventProfiler.cpp new file mode 100644 index 000000000..dbf275523 --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/EventProfiler.cpp @@ -0,0 +1,635 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#include "EventProfiler.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "CuptiEventApi.h" +#include "Logger.h" + +using namespace std::chrono; +using std::accumulate; +using std::endl; +using std::map; +using std::ostream; +using std::string; +using std::unique_ptr; +using std::vector; + +namespace KINETO_NAMESPACE { + +static std::mutex& logMutex() { + static std::mutex instance; + return instance; +} + +// --------------------------------------------------------------------- +// class Event +// --------------------------------------------------------------------- + +// Compute domain instance percentiles +PercentileList& Event::percentiles( + PercentileList& pcs, + const SampleSlice& slice) const { + vector instance_values; + instance_values.reserve(instanceCount); + for (int i = 0; i < instanceCount; i++) { + instance_values.push_back(sumInstance(i, slice)); + } + return KINETO_NAMESPACE::percentiles(instance_values, pcs); +} + +// Add up all samples for a given domain instance +int64_t Event::sumInstance(int i, const SampleSlice& slice) const { + auto r = toIdxRange(slice); + auto start = samples_.cbegin(); + std::advance(start, r.first); + auto end = start; + std::advance(end, r.second); + return accumulate(start, end, 0ul, [i](int64_t a, const Sample& b) { + return a + b.second[i]; + }); +} + +// Add up all samples across all domain instances +int64_t Event::sumAll(const SampleSlice& slice) const { + int64_t res = 0; + for (int i = 0; i < instanceCount; i++) { + res += sumInstance(i, slice); + } + return res; +} + +// Print raw sample values for all domains +void Event::printSamples(ostream& s, CUdevice device) const { + // Don't mess up output with interleaved lines + // Probably OK to reuse logMutex() here since this is + // used for debugging, but need to keep an eye on it. + std::lock_guard lock(logMutex()); + s << "Device " << device << " " << name << ":" << endl; + for (const auto& sample : samples_) { + const auto& vals = sample.second; + for (int64_t val : vals) { + s << val << " "; + } + s << endl; + } +} + +// --------------------------------------------------------------------- +// class Metric +// --------------------------------------------------------------------- +Metric::Metric( + string name, + CUpti_MetricID id, + vector events, + CUpti_MetricEvaluationMode eval_mode, + CuptiMetricApi& cupti_metrics) + : name(std::move(name)), + id_(id), + events_(std::move(events)), + evalMode_(eval_mode), + cuptiMetrics_(cupti_metrics), + valueKind_(cuptiMetrics_.valueKind(id)) {} + +// Return per-SM vector as well as total +struct Metric::CalculatedValues Metric::calculate( + map& event_map, + nanoseconds sample_duration, + const SampleSlice& slice) { + vector metric_values; + vector ev_values; + ev_values.reserve(events_.size()); + if (evalMode_ & CUPTI_METRIC_EVALUATION_MODE_PER_INSTANCE) { + int instance_count = instanceCount(event_map); + metric_values.reserve(instance_count); + for (int i = 0; i < instance_count; i++) { + ev_values.clear(); + for (CUpti_EventID event_id : events_) { + ev_values.push_back(event_map[event_id].sumInstance(i, slice)); + } + metric_values.push_back(cuptiMetrics_.calculate( + id_, valueKind_, events_, ev_values, sample_duration.count())); + } + } + + // FIXME: Check assumption that all instances are profiled + ev_values.clear(); + for (CUpti_EventID event_id : events_) { + ev_values.push_back(event_map[event_id].sumAll(slice)); + } + SampleValue total = cuptiMetrics_.calculate( + id_, valueKind_, events_, ev_values, sample_duration.count()); + if (evalMode_ & CUPTI_METRIC_EVALUATION_MODE_AGGREGATE) { + metric_values.push_back(total); + } + return {metric_values, std::move(total)}; +} + +void Metric::printDescription(ostream& s) const { + s << fmt::format("{} ({})", name, fmt::join(events_, ",")) << endl; +} + +// --------------------------------------------------------------------- +// class EventGroupSet +// --------------------------------------------------------------------- + +// Each domain has a set of counters. +// Some counters in a domain can be collected simultaneously in a "group" +// Counters from different domains can also be collected at the same time +// Therefore we have a "set of groups", or group set, with counters that +// can all be collected at once. +EventGroupSet::EventGroupSet( + CUpti_EventGroupSet& set, + map& events, + CuptiEventApi& cupti) + : set_(set), events_(events), cuptiEvents_(cupti), enabled_(false) { + for (int g = 0; g < set.numEventGroups; g++) { + CUpti_EventGroup grp = set.eventGroups[g]; + // Profile all domain instances + cuptiEvents_.enablePerInstance(grp); + uint32_t instance_count = cuptiEvents_.instanceCount(grp); + for (const auto& id : cuptiEvents_.eventsInGroup(grp)) { + VLOG(0) << "Instance count for " << id << ":" << instance_count; + events_[id].instanceCount = instance_count; + } + } +} + +EventGroupSet::~EventGroupSet() { + // Disable EventGroupSet in Cupti. + if (enabled_) { + setEnabled(false); + } +} + +// Enable or disable this group set +void EventGroupSet::setEnabled(bool enabled) { + if (enabled && !enabled_) { + cuptiEvents_.enableGroupSet(set_); + } else if (!enabled && enabled_) { + cuptiEvents_.disableGroupSet(set_); + } + enabled_ = enabled; +} + +// Collect counter values for each counter in group set +void EventGroupSet::collectSample() { + auto timestamp = system_clock::now(); + for (int g = 0; g < set_.numEventGroups; g++) { + CUpti_EventGroup grp = set_.eventGroups[g]; + for (const auto& id : cuptiEvents_.eventsInGroup(grp)) { + Event& ev = events_[id]; + vector vals(ev.instanceCount); + // FIXME: Use cuptiEventGroupReadAllEvents + cuptiEvents_.readEvent(grp, id, vals); + + if (VLOG_IS_ON(0)) { + for (int64_t v : vals) { + if (v == CUPTI_EVENT_OVERFLOW) { + LOG(WARNING) << "Counter overflow detected " + << "- decrease sample period!" << endl; + } + } + } + + ev.addSample(timestamp, vals); + } + } + + if (VLOG_IS_ON(1)) { + auto t2 = system_clock::now(); + VLOG(1) << "Device " << cuptiEvents_.device() << " Sample (us): " + << duration_cast(t2 - timestamp).count(); + } +} + +// Print names of events in this group set, ordered by group +void EventGroupSet::printDescription(ostream& s) const { + for (int g = 0; g < set_.numEventGroups; g++) { + s << " Events in group " << g << ": "; + for (const auto& id : cuptiEvents_.eventsInGroup(set_.eventGroups[g])) { + s << id << " (" << events_[id].name << ") "; + } + s << endl; + } +} + +// --------------------------------------------------------------------- +// class EventProfiler +// --------------------------------------------------------------------- + +// Find nearest factor of a number by linear search, +// starting at hi and lo - hi searches up and lo searches down +static int nearestFactor(int hi, int lo, int number) { + return number % hi == 0 + ? hi + : number % lo == 0 ? lo : nearestFactor(hi + 1, lo - 1, number); +} + +static int nearestFactor(int count, int max) { + return nearestFactor(count, count, max); +} + +void EventProfiler::initEvents(const std::set& eventNames) { + events_.clear(); + // Build event map + for (const auto& name : eventNames) { + events_.emplace(cuptiEvents_->eventId(name), name); + } +} + +void EventProfiler::initMetrics(const std::set& metricNames) { + metrics_.clear(); + // Add events from metrics + metrics_.reserve(metricNames.size()); + for (const auto& metric_name : metricNames) { + CUpti_MetricID metric_id = cuptiMetrics_->idFromName(metric_name); + if (metric_id == ~0) { + continue; + } + + const auto& events = cuptiMetrics_->events(metric_id); + vector event_ids; + event_ids.reserve(events.size()); + for (const auto& pair : events) { + CUpti_EventID id = pair.first; + const string& event_name = pair.second; + if (event_name.empty()) { + // For unnamed events, use metric name and event id + // FIXME: For subsequent metrics using the same event, + // this will be confusing + events_.emplace(id, metric_name + "_" + event_name); + } else { + events_.emplace(id, event_name); + } + event_ids.push_back(id); + } + metrics_.emplace_back( + metric_name, + metric_id, + event_ids, + cuptiMetrics_->evaluationMode(metric_id), + *cuptiMetrics_); + } +} + +bool EventProfiler::initEventGroups() { + sets_.clear(); + if (eventGroupSets_) { + cuptiEvents_->destroyGroupSets(eventGroupSets_); + eventGroupSets_ = nullptr; + } + if (events_.empty()) { + return true; + } + + // Determine sets of groups to be collected + vector ids; + ids.reserve(events_.size()); + for (const auto& ev : events_) { + ids.push_back(ev.first); + } + eventGroupSets_ = cuptiEvents_->createGroupSets(ids); + VLOG(0) << "Number of group sets: " << eventGroupSets_->numSets; + for (int i = 0; i < eventGroupSets_->numSets; i++) { + sets_.push_back( + EventGroupSet(eventGroupSets_->sets[i], events_, *cuptiEvents_)); + } + return !sets_.empty(); +} + +static unique_ptr alignAndValidateConfigs( + Config& base, + Config* onDemand) { + auto now = system_clock::now(); + if (!onDemand || + now > + (onDemand->eventProfilerOnDemandStartTime() + + onDemand->eventProfilerOnDemandDuration())) { + base.validate(now); + return base.clone(); + } + + auto res = base.clone(); + res->addEvents(onDemand->eventNames()); + res->addMetrics(onDemand->metricNames()); + + int sample_period = + std::min(base.samplePeriod().count(), onDemand->samplePeriod().count()); + if (sample_period < base.samplePeriod().count() && + (base.samplePeriod().count() % sample_period) != 0) { + sample_period = nearestFactor(sample_period, base.samplePeriod().count()); + LOG(WARNING) + << "On-demand sample period must be a factor of base sample period. " + << "Adjusting from " << onDemand->samplePeriod().count() << "ms to " + << sample_period << "ms."; + } + base.setSamplePeriod(milliseconds(sample_period)); + base.validate(now); + res->setSamplePeriod(base.samplePeriod()); + res->setMultiplexPeriod(base.multiplexPeriod()); + res->validate(now); + onDemand->setSamplePeriod(base.samplePeriod()); + onDemand->setMultiplexPeriod(base.multiplexPeriod()); + onDemand->validate(now); + + return res; +} + +static milliseconds minReportPeriod(const Config& config, int num_sets) { + return config.multiplexPeriod() * num_sets; +} + +static bool canSupportReportPeriod(const Config& config, int num_sets) { + // Can we get through the groups an even number per report period? + milliseconds min_report_period = minReportPeriod(config, num_sets); + return (config.reportPeriod().count() % min_report_period.count()) == 0; +} + +static int completeSamplesPerReport(const Config& config, int num_sets) { + if (num_sets <= 1) { + return config.reportPeriod() / config.samplePeriod(); + } + // Numnber of complete sample collections in the report period + // E.g. if report period is 10000ms, sample period 500ms, + // multiplex period 2000ms and num_sets is 5 then # of complete samples is + // (2000ms / 500ms) * (10000ms / 2000ms / 5) = 4 * 1 = 4 + int samples_per_multiplex_period = + config.multiplexPeriod() / config.samplePeriod(); + int multiplex_periods_per_report = + config.reportPeriod() / config.multiplexPeriod(); + return (multiplex_periods_per_report / num_sets) * + samples_per_multiplex_period; +} + +static bool canSupportSamplesPerReport(const Config& config, int num_sets) { + // Can samples per report can be honored with an exact *full* set of samples? + // We don't support partial samples at this point. + int full_samples_per_report = completeSamplesPerReport(config, num_sets); + return (full_samples_per_report % config.samplesPerReport()) == 0; +} + +static void adjustConfig(Config& config, int num_sets) { + // Don't change sample period and multiplex period here, since that can + // cause overflows and perf degradation. Report period and samples per + // report is OK to change (with warning). + if (!canSupportReportPeriod(config, num_sets)) { + milliseconds min_report_period = minReportPeriod(config, num_sets); + LOG(WARNING) << "Report period must be a multiple of " + << min_report_period.count() << "ms (" << num_sets + << " event sets * " << config.multiplexPeriod().count() + << "ms multiplex period), in order to get complete samples."; + auto new_report_period = + Config::alignUp(config.reportPeriod(), min_report_period); + double sf = + ((double)new_report_period.count()) / config.reportPeriod().count(); + int new_samples_per_report = std::round(config.samplesPerReport() * sf); + LOG(WARNING) << "Adjusting report period from " + << config.reportPeriod().count() << "ms to " + << new_report_period.count() << "ms"; + if (new_samples_per_report != config.samplesPerReport()) { + LOG(WARNING) << "Adjusting samples per report from " + << config.samplesPerReport() << " to " + << new_samples_per_report; + } + config.setReportPeriod(new_report_period); + config.setSamplesPerReport(new_samples_per_report); + } + // Ensure that samples per report can be honored with + // an exact *full* set of samples. Don't support partial + // samples at this point. + if (!canSupportSamplesPerReport(config, num_sets)) { + int full_samples_per_report = completeSamplesPerReport(config, num_sets); + int adjusted_count = + nearestFactor(config.samplesPerReport(), full_samples_per_report); + LOG(WARNING) + << "Samples per report must be such that an even number of " + << "complete samples can be aggregated in each report period. Adjusting" + << " from " << config.samplesPerReport() << " to " << adjusted_count + << " (complete sample count is " << full_samples_per_report << ")"; + config.setSamplesPerReport(adjusted_count); + } +} + +// Prepare profiler +EventProfiler::EventProfiler( + std::unique_ptr cupti_events, + std::unique_ptr cupti_metrics, + vector>& loggers, + vector>& onDemandLoggers) + : cuptiEvents_(std::move(cupti_events)), + cuptiMetrics_(std::move(cupti_metrics)), + loggers_(loggers), + onDemandLoggers_(onDemandLoggers) {} + +void EventProfiler::reportSamples() { + dispatchSamples(*config_, loggers_, baseSamples_); + baseSamples_ += completeSamplesPerReport(*config_, sets_.size()); +} + +void EventProfiler::reportOnDemandSamples() { + dispatchSamples(*onDemandConfig_, onDemandLoggers_, onDemandSamples_); + onDemandSamples_ += completeSamplesPerReport(*onDemandConfig_, sets_.size()); +} + +EventProfiler::~EventProfiler() { + if (eventGroupSets_) { + for (auto& set : sets_) { + set.setEnabled(false); + } + cuptiEvents_->destroyGroupSets(eventGroupSets_); + } + VLOG(0) << "Stopped event profiler for device " << device(); +} + +void EventProfiler::updateLoggers(Config& config, Config* on_demand_config) { + // Update loggers. + for (auto& logger : loggers_) { + std::lock_guard lock(logMutex()); + logger->update(config); + } + + if (on_demand_config) { + // Update onDemand loggers. + for (auto& logger : onDemandLoggers_) { + std::lock_guard lock(logMutex()); + logger->update(*on_demand_config); + } + } +} + +bool EventProfiler::applyConfig(const Config& config) { + // Initialize events, metrics, and event group sets. + // TODO: Send warnings / errors back to dyno for onDemand config + try { + if (!initEventsAndMetrics(config)) { + return false; + } + } catch (const std::exception& ex) { + LOG(WARNING) << "Failed to apply config (" << ex.what() << ")"; + return false; + } + + return true; +} + +bool EventProfiler::initEventsAndMetrics(const Config& config) { + initEvents(config.eventNames()); + initMetrics(config.metricNames()); + // We now have the total list of events to collect + // They need to be organized into groups for multiplexing + if (!initEventGroups()) { + LOG(WARNING) << "No events/metrics initialized successfully"; + return false; + } + + if (VLOG_IS_ON(1)) { + printMetrics(LIBKINETO_DBG_STREAM); + printSets(LIBKINETO_DBG_STREAM); + } + return true; +} + +void EventProfiler::printSets(ostream& s) const { + for (int i = 0; i < sets_.size(); i++) { + s << "Set " << i << endl; + sets_[i].printDescription(s); + } +} + +void EventProfiler::printMetrics(ostream& s) const { + s << "Metrics:" << endl; + for (const Metric& m : metrics_) { + m.printDescription(s); + } +} + +void EventProfiler::printAllSamples(ostream& s, CUdevice device) const { + for (const auto& pair : events_) { + const Event& ev = pair.second; + ev.printSamples(s, device); + } +} + +void EventProfiler::enableNextCounterSet() { + if (sets_.size() > 1) { + auto t1 = system_clock::now(); + + VLOG(1) << "Disabling set " << curEnabledSet_; + sets_[curEnabledSet_].setEnabled(false); + curEnabledSet_ = (curEnabledSet_ + 1) % sets_.size(); + VLOG(1) << "Enabling set " << curEnabledSet_; + sets_[curEnabledSet_].setEnabled(true); + + if (VLOG_IS_ON(1)) { + auto t2 = system_clock::now(); + VLOG(1) << "Switch (us): " + << duration_cast(t2 - t1).count(); + } + } +} + +// Notify listeners of collected samples +void EventProfiler::dispatchSamples( + const Config& config, + const vector>& loggers, + int sample_offset) { + Sample sample(events_.size() + metrics_.size()); + // Normalize values to per second + auto delta = config.reportPeriod() / config.samplesPerReport(); + double sf = 1000.0 * sets_.size() / delta.count(); + for (int i = 0; i < config.samplesPerReport(); i++) { + sample.stats.clear(); + sample.deltaMsec = (delta * i).count(); + SampleSlice slice = {sample_offset, i, config.samplesPerReport()}; + VLOG(1) << "Slice: " << sample_offset << ", " << i << ", " + << config.samplesPerReport(); + for (const auto& pair : events_) { + const Event& ev = pair.second; + int64_t total = std::round(sf * ev.sumAll(slice)); + PercentileList pcs = initPercentiles(config.percentiles()); + normalize(ev.percentiles(pcs, slice), sf); + sample.stats.push_back({ev.name, std::move(pcs), SampleValue(total)}); + } + + for (auto& m : metrics_) { + // calculate returns a pair of per-SM vector and a total + auto vals = m.calculate(events_, delta, slice); + PercentileList pcs = initPercentiles(config.percentiles()); + sample.stats.push_back( + {m.name, std::move(percentiles(vals.perInstance, pcs)), vals.total}); + } + + for (auto& logger : loggers) { + std::lock_guard lock(logMutex()); + logger->handleSample(device(), sample, config.ipcFabricEnabled()); + } + } + + if (VLOG_IS_ON(2)) { + printAllSamples(LIBKINETO_DBG_STREAM, device()); + } +} + +void EventProfiler::configure(Config& config, Config* onDemandConfig) { + if (!sets_.empty()) { + sets_[curEnabledSet_].setEnabled(false); + clearSamples(); + } + + config_ = config.clone(); + onDemandConfig_ = onDemandConfig ? onDemandConfig->clone() : nullptr; + mergedConfig_ = alignAndValidateConfigs(*config_, onDemandConfig_.get()); + if (!applyConfig(*mergedConfig_)) { + LOG(WARNING) << "Failed to apply config!"; + mergedConfig_ = config_->clone(); + applyConfig(*config_); + } + if (!sets_.empty()) { + // Make timing adjustments based on multiplexing requirements. + adjustConfig(*config_, sets_.size()); + if (onDemandConfig_) { + int duration = onDemandConfig_->eventProfilerOnDemandDuration().count(); + LOG(INFO) << "On demand profiler activated for " << duration << " secs"; + adjustConfig(*onDemandConfig_, sets_.size()); + } + // If events or metrics were added or removed, need to tell loggers + updateLoggers(*config_, onDemandConfig_.get()); + } + + curEnabledSet_ = 0; + if (!sets_.empty()) { + sets_[0].setEnabled(true); + } else { + VLOG(0) << "No counters profiled!"; + } + + baseSamples_ = 0; + onDemandSamples_ = 0; +} + +void EventProfiler::collectSample() { + if (sets_.empty()) { + return; + } + sets_[curEnabledSet_].collectSample(); + if (VLOG_IS_ON(1)) { + printAllSamples(LIBKINETO_DBG_STREAM, device()); + } +} + +} // namespace KINETO_NAMESPACE diff --git a/tb_plugins/profiling/libkineto/src/EventProfiler.h b/tb_plugins/profiling/libkineto/src/EventProfiler.h new file mode 100644 index 000000000..fafd5b9bb --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/EventProfiler.h @@ -0,0 +1,341 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "Config.h" +#include "CuptiEventApi.h" +#include "CuptiMetricApi.h" +#include "SampleListener.h" + +namespace KINETO_NAMESPACE { + +// Helper function for computing percentiles (nearest-rank). +// Modifies the input. +template +inline PercentileList& percentiles(std::vector values, PercentileList& pcs) { + auto size = values.size(); + for (auto& x : pcs) { + int idx = std::min(size - 1, (x.first * size) / 100); + std::nth_element(values.begin(), values.begin() + idx, values.end()); + x.second = SampleValue(values[idx]); + } + return pcs; +} + +// Helper function for normalizing a percentile list +// Modifies the input +inline PercentileList& normalize(PercentileList& pcs, double sf) { + for (auto& pc : pcs) { + pc.second *= sf; + } + return pcs; +} + +// A slice of the sample buffer +struct SampleSlice { + // Start offset (samples) + int offset; + // Slice number + int index; + // Out of this many + int count; +}; + +// A sampled event +class Event { + public: + /* implicit */ Event(std::string name) : name(std::move(name)) {} + /* implicit */ Event(const char* name) : name(name) {} + Event() : name("INVALID") {} + + Event(const Event&) = delete; + Event& operator=(const Event&) = delete; + Event(Event&&) = default; + Event& operator=(Event&&) = default; + + void addSample( + std::chrono::time_point timestamp, + const std::vector& values) { + assert(values.size() == instanceCount); + samples_.emplace_back(timestamp, values); + } + + // Sum samples for a single domain instance + int64_t sumInstance(int i, const SampleSlice& slice) const; + + // Sum all samples across all domain instances + int64_t sumAll(const SampleSlice& slice) const; + + // Create list of percentiles + PercentileList& percentiles(PercentileList& pcs, const SampleSlice& slice) + const; + + void eraseSamples(int count) { + auto end = samples_.begin(); + std::advance(end, count); + samples_.erase(samples_.begin(), end); + } + + void clearSamples() { + samples_.clear(); + } + + int sampleCount() { + return samples_.size(); + } + + void printSamples(std::ostream& s, CUdevice device) const; + + // Event name (see nvprof --query-events) + std::string name; + + // Number of domain instances for this event, e.g. number of SMs + int instanceCount = 0; + + private: + std::pair toIdxRange(const SampleSlice& slice) const { + int size = (samples_.size() - slice.offset) / slice.count; + return std::make_pair(slice.offset + (slice.index * size), size); + } + + // List of collected samples, where each sample has values for + // one or more domain instances + using Sample = std::pair< + std::chrono::time_point, + std::vector>; + std::list samples_; +}; + +class Metric { + public: + Metric( + std::string name, + CUpti_MetricID id, + std::vector events, + CUpti_MetricEvaluationMode eval_mode, + CuptiMetricApi& cupti_metrics); + + struct CalculatedValues { + std::vector perInstance; + SampleValue total; + }; + + struct CalculatedValues calculate( + std::map& events, + std::chrono::nanoseconds sample_duration, + const SampleSlice& slice); + + int instanceCount(std::map& events) { + return events[events_[0]].instanceCount; + } + + void printDescription(std::ostream& s) const; + + std::string name; + + private: + CUpti_MetricID id_; + std::vector events_; + CUpti_MetricEvaluationMode evalMode_; + // Calls to CUPTI is encapsulated behind this interface + CuptiMetricApi& cuptiMetrics_; + CUpti_MetricValueKind valueKind_; +}; + +/** + * A set of event groups. + * Holds all the events that may be collected in a single pass. + * A group contains one or more counters for a single domain. + * A group set contains zero or one groups per domain. + */ +class EventGroupSet { + public: + EventGroupSet( + CUpti_EventGroupSet& set, + std::map& events, + CuptiEventApi& cupti); + ~EventGroupSet(); + + EventGroupSet(const EventGroupSet&) = delete; + EventGroupSet& operator=(const EventGroupSet&) = delete; + EventGroupSet(EventGroupSet&&) = default; + EventGroupSet& operator=(EventGroupSet&&) = delete; + + // Number of groups = number of domains profiled + int groupCount() const { + return set_.numEventGroups; + } + + void setEnabled(bool enabled); + // Take a sample of counters in this group set + void collectSample(); + void printDescription(std::ostream& s) const; + + private: + CUpti_EventGroupSet& set_; + std::map& events_; + // Calls to CUPTI is encapsulated behind this interface + CuptiEventApi& cuptiEvents_; + bool enabled_; +}; + +// The sampler +class EventProfiler { + public: + explicit EventProfiler( + std::unique_ptr cupti_events, + std::unique_ptr cupti_metrics, + std::vector>& loggers, + std::vector>& onDemandLoggers); + EventProfiler(const EventProfiler&) = delete; + EventProfiler& operator=(const EventProfiler&) = delete; + ~EventProfiler(); + + void configure(Config& config, Config* onDemandConfig); + + bool isOnDemandActive() { + return !!onDemandConfig_; + } + + // Print the counter sets. Multiple sets will be multiplexed. + void printSets(std::ostream& s) const; + + // Print metrics descriptions + void printMetrics(std::ostream& s) const; + + bool enableForDevice(Config& cfg); + + CUdevice device() { + return cuptiEvents_->device(); + } + + bool setContinuousMode() { + return cuptiEvents_->setContinuousMode(); + } + + std::chrono::milliseconds samplePeriod() { + return mergedConfig_->samplePeriod(); + } + + std::chrono::milliseconds multiplexPeriod() { + return mergedConfig_->multiplexPeriod(); + } + + std::chrono::milliseconds reportPeriod() { + return config_->reportPeriod(); + } + + std::chrono::milliseconds onDemandReportPeriod() { + return onDemandConfig_->reportPeriod(); + } + + // Read values of currently running counters. + void collectSample(); + + void reportSamples(); + void reportOnDemandSamples(); + + bool enabled() { + return sets_.size() > 0; + } + + bool multiplexEnabled() { + return sets_.size() > 1; + } + + // Multiplex counters. + void enableNextCounterSet(); + + void eraseReportedSamples() { + int erase_count = baseSamples_; + if (onDemandConfig_ && + onDemandConfig_->eventProfilerOnDemandDuration().count() > 0) { + erase_count = std::min(baseSamples_, onDemandSamples_); + } + eraseSamples(erase_count); + baseSamples_ -= erase_count; + onDemandSamples_ -= erase_count; + } + + void clearSamples() { + for (auto& pair : events_) { + pair.second.clearSamples(); + } + baseSamples_ = 0; + onDemandSamples_ = 0; + } + + private: + // Functions to initialize profiler based on Config settings. + bool applyConfig(const Config& config); + bool initEventsAndMetrics(const Config& config); + void initEvents(const std::set& eventNames); + void initMetrics(const std::set& metricNames); + bool initEventGroups(); + + PercentileList initPercentiles(const std::vector& percentiles) { + PercentileList res; + res.reserve(percentiles.size()); + for (int p : percentiles) { + res.emplace_back(p, SampleValue(0)); + } + return res; + } + + // Notify listeners of collected samples + void dispatchSamples( + const Config& config, + const std::vector>& loggers, + int report_nr); + + void eraseSamples(int count) { + for (auto& pair : events_) { + pair.second.eraseSamples(count); + } + } + + void updateLoggers(Config& config, Config* on_demand_config); + + // Print all collected samples since last clear. + void printAllSamples(std::ostream& s, CUdevice device) const; + + // Calls to CUPTI is encapsulated behind these interfaces + std::unique_ptr cuptiEvents_; + std::unique_ptr cuptiMetrics_; + // The CUpti API reports event IDs, we must map them to our event objects + std::map events_; + // List of metrics + std::vector metrics_; + // The countert sets needed to collect all counters + std::vector sets_; + // The event group set object returned by Cupti. + // Saved s.t. we can call cuptiEventGroupSetsDestroy to free memory when + // the object is no longer needed. + CUpti_EventGroupSets* eventGroupSets_ = nullptr; + // Current multiplexed counter set + int curEnabledSet_{0}; + + std::unique_ptr config_; + std::unique_ptr onDemandConfig_; + std::unique_ptr mergedConfig_; + int baseSamples_{0}; + int onDemandSamples_{0}; + + // Shared between profiler threads + // Vectors are read-only but calling loggers require lock + const std::vector>& loggers_; + const std::vector>& onDemandLoggers_; +}; + +} // namespace KINETO_NAMESPACE diff --git a/tb_plugins/profiling/libkineto/src/EventProfilerController.cpp b/tb_plugins/profiling/libkineto/src/EventProfilerController.cpp new file mode 100644 index 000000000..0427cc7a9 --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/EventProfilerController.cpp @@ -0,0 +1,423 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#include "EventProfilerController.h" + +#include +#include +#include + +#include "ConfigLoader.h" +#include "CuptiEventApi.h" +#include "CuptiMetricApi.h" +#include "EventProfiler.h" +#include "output_csv.h" + +#include "Logger.h" +#include "ThreadUtil.h" + +using namespace std::chrono; +using std::unique_ptr; +using std::vector; + +namespace KINETO_NAMESPACE { + +namespace { + +vector(const Config&)>>& +loggerFactories() { + static vector(const Config&)>> + factories; + return factories; +} + +vector(const Config&)>>& +onDemandLoggerFactories() { + static vector(const Config&)>> + factories; + return factories; +} + +vector> makeLoggers(const Config& config) { + vector> loggers; + for (const auto& factory : loggerFactories()) { + loggers.push_back(factory(config)); + } + loggers.push_back(std::make_unique()); + loggers.push_back(std::make_unique()); + return loggers; +} + +vector> makeOnDemandLoggers( + const Config& config) { + vector> loggers; + for (const auto& factory : onDemandLoggerFactories()) { + loggers.push_back(factory(config)); + } + loggers.push_back(std::make_unique()); + return loggers; +} + +vector>& loggers(const Config& config) { + static auto res = makeLoggers(config); + return res; +} + +vector>& onDemandLoggers( + const Config& config) { + static auto res = makeOnDemandLoggers(config); + return res; +} + +} // anon namespace + +// Keep an eye on profiling threads. +// We've observed deadlocks in Cuda11 in libcuda / libcupti.. +namespace detail { + +class HeartbeatMonitor { + + public: + ~HeartbeatMonitor() { + stopMonitoring(); + } + + static HeartbeatMonitor& instance() { + static HeartbeatMonitor monitor; + return monitor; + } + + void profilerHeartbeat() { + int32_t tid = systemThreadId(); + std::lock_guard lock(mutex_); + profilerAliveMap_[tid]++; + } + + void setPeriod(seconds period) { + { + std::lock_guard lock(mutex_); + if (period_ == period) { + return; + } + period_ = period; + } + if (period == seconds(0)) { + stopMonitoring(); + } else { + startMonitoring(); + } + } + + private: + HeartbeatMonitor() = default; + + void monitorLoop() { + std::unique_lock lock(mutex_); + while(!stopMonitor_) { + auto cv_status = condVar_.wait_for(lock, seconds(period_)); + // Don't perform check on spurious wakeup or on notify + if (cv_status == std::cv_status::timeout) { + for (auto& pair : profilerAliveMap_) { + int32_t tid = pair.first; + int& i = pair.second; + if (i == 0) { + LOG(ERROR) << "Thread " << tid << " appears stuck!"; + } + i = 0; + } + } + } + } + + void startMonitoring() { + if (!monitorThread_) { + VLOG(0) << "Starting monitoring thread"; + stopMonitor_ = false; + monitorThread_ = std::make_unique( + &HeartbeatMonitor::monitorLoop, this); + } + } + + void stopMonitoring() { + if (monitorThread_) { + VLOG(0) << "Stopping monitoring thread"; + stopMonitor_ = true; + condVar_.notify_one(); + monitorThread_->join(); + monitorThread_ = nullptr; + VLOG(0) << "Monitoring thread terminated"; + } + } + + std::map profilerAliveMap_; + std::unique_ptr monitorThread_; + std::mutex mutex_; + std::condition_variable condVar_; + std::atomic_bool stopMonitor_{false}; + seconds period_{0}; +}; + +} // namespace detail + +namespace { +// Profiler map singleton +std::map>& profilerMap() { + static std::map> instance; + return instance; +} + +void reportLateSample( + int sleepMs, + int sampleMs, + int reportMs, + int reprogramMs) { + LOG_EVERY_N(WARNING, 10) << "Lost sample due to delays (ms): " << sleepMs + << ", " << sampleMs << ", " << reportMs << ", " + << reprogramMs; +} + +void configureHeartbeatMonitor( + detail::HeartbeatMonitor& monitor, const Config& base, const Config* onDemand) { + seconds base_period = + base.eventProfilerHeartbeatMonitorPeriod(); + seconds on_demand_period = !onDemand ? seconds(0) : + onDemand->eventProfilerHeartbeatMonitorPeriod(); + monitor.setPeriod( + on_demand_period > seconds(0) ? on_demand_period : base_period); +} + +} // anon namespace + +void EventProfilerController::addLoggerFactory( + std::function(const Config&)> factory) { + loggerFactories().push_back(factory); +} + +void EventProfilerController::addOnDemandLoggerFactory( + std::function(const Config&)> factory) { + onDemandLoggerFactories().push_back(factory); +} + +EventProfilerController::EventProfilerController( + CUcontext context, + ConfigLoader& configLoader, + detail::HeartbeatMonitor& heartbeatMonitor) + : configLoader_(configLoader), heartbeatMonitor_(heartbeatMonitor) { + auto cupti_events = std::make_unique(context); + auto cupti_metrics = + std::make_unique(cupti_events->device()); + configLoader_.addHandler( + ConfigLoader::ConfigKind::EventProfiler, this); + auto config = configLoader.getConfigCopy(); + profiler_ = std::make_unique( + std::move(cupti_events), + std::move(cupti_metrics), + loggers(*config), + onDemandLoggers(*config)); + profilerThread_ = std::make_unique( + &EventProfilerController::profilerLoop, this); +} + +EventProfilerController::~EventProfilerController() { + if (profilerThread_) { + // signaling termination of the profiler loop + stopRunloop_ = true; + profilerThread_->join(); + } + configLoader_.removeHandler( + ConfigLoader::ConfigKind::EventProfiler, this); + VLOG(0) << "Stopped event profiler"; +} + +// Must be called under lock +void EventProfilerController::start(CUcontext ctx, ConfigLoader& configLoader) { + profilerMap()[ctx] = unique_ptr( + new EventProfilerController( + ctx, configLoader, detail::HeartbeatMonitor::instance())); +} + +// Must be called under lock +void EventProfilerController::stop(CUcontext ctx) { + profilerMap()[ctx] = nullptr; +} + +bool EventProfilerController::canAcceptConfig() { + std::lock_guard guard(mutex_); + return !newOnDemandConfig_; +} + +void EventProfilerController::acceptConfig(const Config& config) { + if (config.eventProfilerOnDemandDuration().count() == 0) { + // Ignore - not for this profiler + return; + } + std::lock_guard guard(mutex_); + if (newOnDemandConfig_) { + LOG(ERROR) << "On demand request already queued - ignoring new request"; + return; + } + newOnDemandConfig_ = config.clone(); + LOG(INFO) << "Received new on-demand config"; +} + +bool EventProfilerController::enableForDevice(Config& cfg) { + // FIXME: Use device unique id! + if (!cfg.eventProfilerEnabledForDevice(profiler_->device())) { + return false; + } + // context count includes the new context + int instances = configLoader_.contextCountForGpu(profiler_->device()); + VLOG(0) << "Device context count: " << instances; + return instances >= 0 && instances <= cfg.maxEventProfilersPerGpu(); +} + +void EventProfilerController::profilerLoop() { + // We limit the number of profilers that can exist per GPU + auto config = configLoader_.getConfigCopy(); + if (!enableForDevice(*config)) { + VLOG(0) << "Not starting EventProfiler - profilers for GPU " + << profiler_->device() << " exceeds profilers per GPU limit (" + << config->maxEventProfilersPerGpu() << ")"; + return; + } + + if (!profiler_->setContinuousMode()) { + VLOG(0) << "Continuous mode not supported for GPU " + << profiler_->device() << ". Not starting Event Profiler."; + return; + } + + VLOG(0) << "Starting Event Profiler for GPU " << profiler_->device(); + setThreadName("CUPTI Event Profiler"); + + time_point next_sample_time; + time_point next_report_time; + time_point next_on_demand_report_time; + time_point next_multiplex_time; + std::unique_ptr on_demand_config = nullptr; + bool reconfigure = true; + bool restart = true; + int report_count = 0; + int on_demand_report_count = 0; + while (!stopRunloop_) { + heartbeatMonitor_.profilerHeartbeat(); + if (configLoader_.hasNewConfig(*config)) { + config = configLoader_.getConfigCopy(); + VLOG(0) << "Base config changed"; + report_count = 0; + reconfigure = true; + } + + auto now = system_clock::now(); + if (on_demand_config && + now > (on_demand_config->eventProfilerOnDemandStartTime() + + on_demand_config->eventProfilerOnDemandDuration())) { + on_demand_config = nullptr; + LOG(INFO) << "On-demand profiling complete"; + reconfigure = true; + } + + if (!profiler_->isOnDemandActive()) { + std::lock_guard lock(mutex_); + if (newOnDemandConfig_) { + VLOG(0) << "Received on-demand config, reconfiguring"; + on_demand_config = std::move(newOnDemandConfig_); + reconfigure = true; + on_demand_report_count = 0; + } + } + + if (reconfigure) { + try { + profiler_->configure(*config, on_demand_config.get()); + } catch (const std::exception& ex) { + LOG(ERROR) << "Encountered error while configuring event profiler: " + << ex.what(); + // Exit profiling entirely when encountering an error here + // as it indicates a serious problem or bug. + break; + } + configureHeartbeatMonitor( + heartbeatMonitor_, *config, on_demand_config.get()); + reconfigure = false; + restart = true; + } + + if (restart) { + now = system_clock::now(); + next_sample_time = now + profiler_->samplePeriod(); + next_report_time = now + profiler_->reportPeriod(); + if (profiler_->isOnDemandActive()) { + next_on_demand_report_time = now + profiler_->onDemandReportPeriod(); + } + next_multiplex_time = now + profiler_->multiplexPeriod(); + // Collect an initial sample and throw it away + // The next sample is the first valid one + profiler_->collectSample(); + profiler_->clearSamples(); + restart = false; + } + + auto start_sleep = now; + while (now < next_sample_time) { + /* sleep override */ + std::this_thread::sleep_for(next_sample_time - now); + now = system_clock::now(); + } + int sleep_time = duration_cast(now - start_sleep).count(); + + auto start_sample = now; + profiler_->collectSample(); + now = system_clock::now(); + int sample_time = duration_cast(now - start_sample).count(); + + next_sample_time += profiler_->samplePeriod(); + if (now > next_sample_time) { + reportLateSample(sleep_time, sample_time, 0, 0); + restart = true; + continue; + } + + auto start_report = now; + if (now > next_report_time) { + VLOG(1) << "Report #" << report_count++; + profiler_->reportSamples(); + next_report_time += profiler_->reportPeriod(); + } + if (profiler_->isOnDemandActive() && now > next_on_demand_report_time) { + VLOG(1) << "OnDemand Report #" << on_demand_report_count++; + profiler_->reportOnDemandSamples(); + next_on_demand_report_time += profiler_->onDemandReportPeriod(); + } + profiler_->eraseReportedSamples(); + now = system_clock::now(); + int report_time = duration_cast(now - start_report).count(); + + if (now > next_sample_time) { + reportLateSample(sleep_time, sample_time, report_time, 0); + restart = true; + continue; + } + + auto start_multiplex = now; + if (profiler_->multiplexEnabled() && now > next_multiplex_time) { + profiler_->enableNextCounterSet(); + next_multiplex_time += profiler_->multiplexPeriod(); + } + now = system_clock::now(); + int multiplex_time = + duration_cast(now - start_multiplex).count(); + + if (now > next_sample_time) { + reportLateSample(sleep_time, sample_time, report_time, multiplex_time); + restart = true; + } + + VLOG(0) << "Runloop execution time: " + << duration_cast(now - start_sample).count() << "ms"; + } + + VLOG(0) << "Device " << profiler_->device() + << ": Exited event profiling loop"; +} + +} // namespace KINETO_NAMESPACE diff --git a/tb_plugins/profiling/libkineto/src/EventProfilerController.h b/tb_plugins/profiling/libkineto/src/EventProfilerController.h new file mode 100644 index 000000000..007a82faa --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/EventProfilerController.h @@ -0,0 +1,63 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#pragma once + +#include +#include +#include +#include +#include + +#include + +#include "ConfigLoader.h" + +namespace KINETO_NAMESPACE { + +class Config; +class ConfigLoader; +class EventProfiler; +class SampleListener; + +namespace detail { +class HeartbeatMonitor; +} + +class EventProfilerController : public ConfigLoader::ConfigHandler { + public: + EventProfilerController(const EventProfilerController&) = delete; + EventProfilerController& operator=(const EventProfilerController&) = delete; + + ~EventProfilerController(); + + static void start(CUcontext ctx, ConfigLoader& configLoader); + static void stop(CUcontext ctx); + + static void addLoggerFactory( + std::function(const Config&)> factory); + + static void addOnDemandLoggerFactory( + std::function(const Config&)> factory); + + bool canAcceptConfig() override; + + void acceptConfig(const Config& config) override; + + private: + explicit EventProfilerController( + CUcontext context, + ConfigLoader& configLoader, + detail::HeartbeatMonitor& heartbeatMonitor); + bool enableForDevice(Config& cfg); + void profilerLoop(); + + ConfigLoader& configLoader_; + std::unique_ptr newOnDemandConfig_; + detail::HeartbeatMonitor& heartbeatMonitor_; + std::unique_ptr profiler_; + std::unique_ptr profilerThread_; + std::atomic_bool stopRunloop_{false}; + std::mutex mutex_; +}; + +} // namespace KINETO_NAMESPACE diff --git a/tb_plugins/profiling/libkineto/src/GenericTraceActivity.cpp b/tb_plugins/profiling/libkineto/src/GenericTraceActivity.cpp new file mode 100644 index 000000000..4e00b1256 --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/GenericTraceActivity.cpp @@ -0,0 +1,10 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#include "GenericTraceActivity.h" +#include "output_base.h" + +namespace libkineto { + void GenericTraceActivity::log(ActivityLogger& logger) const { + logger.handleGenericActivity(*this); + } +} // namespace libkineto diff --git a/tb_plugins/profiling/libkineto/src/ILoggerObserver.cpp b/tb_plugins/profiling/libkineto/src/ILoggerObserver.cpp new file mode 100644 index 000000000..f01065788 --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/ILoggerObserver.cpp @@ -0,0 +1,54 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +// TODO(T90238193) +// @lint-ignore-every CLANGTIDY facebook-hte-RelativeInclude +#include "ILoggerObserver.h" + +#if !USE_GOOGLE_LOG + +#include +#include + +namespace libkineto { + +struct LoggerTypeName { + constexpr LoggerTypeName(const char* n, LoggerOutputType t) : name(n), type(t) {}; + const char* name; + LoggerOutputType type; +}; + +static constexpr std::array LoggerMap{{ + {"VERBOSE", LoggerOutputType::VERBOSE}, + {"INFO", LoggerOutputType::INFO}, + {"WARNING", LoggerOutputType::WARNING}, + {"ERROR", LoggerOutputType::ERROR}, + {"STAGE", LoggerOutputType::STAGE}, + {"???", LoggerOutputType::ENUM_COUNT} +}}; + +static constexpr bool matchingOrder(int idx = 0) { + return LoggerMap[idx].type == LoggerOutputType::ENUM_COUNT || + ((idx == (int) LoggerMap[idx].type) && matchingOrder(idx + 1)); +} +static_assert(matchingOrder(), "LoggerTypeName map is out of order"); + +const char* toString(LoggerOutputType t) { + if(t < VERBOSE || t >= ENUM_COUNT) { + return LoggerMap[ENUM_COUNT].name; + } + return LoggerMap[(int)t].name; +} + +LoggerOutputType toLoggerOutputType(const std::string& str) { + for (int i = 0; i < LoggerTypeCount; i++) { + if (str == LoggerMap[i].name) { + return LoggerMap[i].type; + } + } + throw std::invalid_argument(fmt::format("Invalid activity type: {}", str)); +} + +} // namespace libkineto + + +#endif // !USE_GOOGLE_LOG diff --git a/tb_plugins/profiling/libkineto/src/Logger.cpp b/tb_plugins/profiling/libkineto/src/Logger.cpp new file mode 100644 index 000000000..dbde765f5 --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/Logger.cpp @@ -0,0 +1,136 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +// TODO(T90238193) +// @lint-ignore-every CLANGTIDY facebook-hte-RelativeInclude +#include "Logger.h" +#include "ILoggerObserver.h" + +#ifndef USE_GOOGLE_LOG + +#include +#include +#include +#include +#include + +#include +#include + +#include "ThreadUtil.h" + +namespace KINETO_NAMESPACE { + +std::atomic_int Logger::severityLevel_{VERBOSE}; +std::atomic_int Logger::verboseLogLevel_{-1}; +std::atomic Logger::verboseLogModules_{~0ull}; + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wglobal-constructors" +std::mutex Logger::loggerObserversMutex_; +#pragma GCC diagnostic pop + + +Logger::Logger(int severity, int line, const char* filePath, int errnum) + : buf_(), out_(LIBKINETO_DBG_STREAM), errnum_(errnum), messageSeverity_(severity) { + buf_ << toString((LoggerOutputType) severity) << ":"; + + const auto tt = + std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); + const char* file = strrchr(filePath, '/'); + buf_ << fmt::format("{:%Y-%m-%d %H:%M:%S}", fmt::localtime(tt)) << " " + << processId() << ":" << systemThreadId() << " " + << (file ? file + 1 : filePath) << ":" << line << "] "; +} + +Logger::~Logger() { +#ifdef __linux__ + if (errnum_ != 0) { + thread_local char buf[1024]; + buf_ << " : " << strerror_r(errnum_, buf, sizeof(buf)); + } +#endif + + { + std::lock_guard guard(loggerObserversMutex_); + for (auto* observer : loggerObservers()) { + // Output to observers. Current Severity helps keep track of which bucket the output goes. + if (observer) { + observer->write(buf_.str(), (LoggerOutputType) messageSeverity_); + } + } + } + + // Finally, print to terminal or console. + out_ << buf_.str() << std::endl; +} + +void Logger::setVerboseLogModules(const std::vector& modules) { + uint64_t mask = 0; + if (modules.empty()) { + mask = ~0ull; + } else { + for (const std::string& name : modules) { + mask |= hash(name.c_str()); + } + } + verboseLogModules_ = mask; +} + +void Logger::addLoggerObserver(ILoggerObserver* observer) { + if (observer == nullptr) { + return; + } + std::lock_guard guard(loggerObserversMutex_); + loggerObservers().insert(observer); +} + +void Logger::removeLoggerObserver(ILoggerObserver* observer) { + std::lock_guard guard(loggerObserversMutex_); + loggerObservers().erase(observer); +} + +void Logger::addLoggerObserverDevice(int64_t device) { + std::lock_guard guard(loggerObserversMutex_); + for (auto observer : loggerObservers()) { + observer->addDevice(device); + } +} + +void Logger::addLoggerObserverEventCount(int64_t count) { + std::lock_guard guard(loggerObserversMutex_); + for (auto observer : loggerObservers()) { + observer->addEventCount(count); + } +} + +void Logger::setLoggerObserverTraceDurationMS(int64_t duration) { + std::lock_guard guard(loggerObserversMutex_); + for (auto observer : loggerObservers()) { + observer->setTraceDurationMS(duration); + } +} + +void Logger::setLoggerObserverTraceID(const std::string& tid) { + std::lock_guard guard(loggerObserversMutex_); + for (auto observer : loggerObservers()) { + observer->setTraceID(tid); + } +} + +void Logger::setLoggerObserverGroupTraceID(const std::string& gtid) { + std::lock_guard guard(loggerObserversMutex_); + for (auto observer : loggerObservers()) { + observer->setGroupTraceID(gtid); + } +} + +void Logger::addLoggerObserverDestination(const std::string& dest) { + std::lock_guard guard(loggerObserversMutex_); + for (auto observer : loggerObservers()) { + observer->addDestination(dest); + } +} + +} // namespace KINETO_NAMESPACE + +#endif // USE_GOOGLE_LOG diff --git a/tb_plugins/profiling/libkineto/src/Logger.h b/tb_plugins/profiling/libkineto/src/Logger.h new file mode 100644 index 000000000..868fc84b9 --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/Logger.h @@ -0,0 +1,244 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#pragma once + +#include + +#define LIBKINETO_DBG_STREAM std::cerr + +#if USE_GOOGLE_LOG + +#include + +#define SET_LOG_SEVERITY_LEVEL(level) +#define SET_LOG_VERBOSITY_LEVEL(level, modules) +#define LOGGER_OBSERVER_ADD_DEVICE(device) +#define LOGGER_OBSERVER_ADD_EVENT_COUNT(count) +#define LOGGER_OBSERVER_SET_TRACE_DURATION_MS(duration) +#define LOGGER_OBSERVER_SET_TRACE_ID(tid) +#define LOGGER_OBSERVER_SET_GROUP_TRACE_ID(gtid) +#define LOGGER_OBSERVER_ADD_DESTINATION(dest) +#define UST_LOGGER_MARK_COMPLETED(stage) + +#else // !USE_GOOGLE_LOG +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// TODO(T90238193) +// @lint-ignore-every CLANGTIDY facebook-hte-RelativeInclude +#include "ILoggerObserver.h" + +#ifdef _MSC_VER +// unset a predefined ERROR (windows) +#undef ERROR +#endif // _MSC_VER + +namespace KINETO_NAMESPACE { + +class Logger { + public: + Logger(int severity, int line, const char* filePath, int errnum = 0); + ~Logger(); + + inline std::ostream& stream() { + return buf_; + } + + static inline void setSeverityLevel(int level) { + severityLevel_ = level; + } + + static inline int severityLevel() { + return severityLevel_; + } + + static inline void setVerboseLogLevel(int level) { + verboseLogLevel_ = level; + } + + static inline int verboseLogLevel() { + return verboseLogLevel_; + } + + // This is constexpr so that the hash for a file name is computed at compile + // time when used in the VLOG macros. + // This way, there is no string comparison for matching VLOG modules, + // only a comparison of pre-computed hashes. + // No fancy hashing needed here. It's pretty inefficient (one character + // at a time) but the strings are not large and it's not in the critical path. + static constexpr uint64_t rol(uint64_t val, int amount) { + return val << amount | val >> (63 - amount); + } + static constexpr uint64_t hash(const char* s) { + uint64_t hash = hash_rec(s, 0); + return hash & rol(0x41a0240682483014ull, hash & 63); + } + static constexpr uint64_t hash_rec(const char* s, int off) { + // Random constants! + return (!s[off] ? 57ull : (hash_rec(s, off + 1) * 293) ^ s[off]); + } + static constexpr const char* basename(const char* s, int off = 0) { + return !s[off] + ? s + : s[off] == '/' ? basename(&s[off + 1]) : basename(s, off + 1); + } + + static void setVerboseLogModules(const std::vector& modules); + + static inline uint64_t verboseLogModules() { + return verboseLogModules_; + } + + static void clearLoggerObservers() { + std::lock_guard g(loggerObserversMutex_); + loggerObservers().clear(); + } + + static void addLoggerObserver(ILoggerObserver* observer); + + static void removeLoggerObserver(ILoggerObserver* observer); + + static void addLoggerObserverDevice(int64_t device); + + static void addLoggerObserverEventCount(int64_t count); + + static void setLoggerObserverTraceDurationMS(int64_t duration); + + static void setLoggerObserverTraceID(const std::string& tid); + + static void setLoggerObserverGroupTraceID(const std::string& gtid); + + static void addLoggerObserverDestination(const std::string& dest); + + private: + std::stringstream buf_; + std::ostream& out_; + int errnum_; + int messageSeverity_; + static std::atomic_int severityLevel_; + static std::atomic_int verboseLogLevel_; + static std::atomic verboseLogModules_; + static std::set& loggerObservers() { + static auto* inst = new std::set(); + return *inst; + } + static std::mutex loggerObserversMutex_; +}; + +class VoidLogger { + public: + VoidLogger() {} + void operator&(std::ostream&) {} +}; + +} // namespace KINETO_NAMESPACE + +#ifdef LOG // Undefine in case these are already defined (quite likely) +#undef LOG +#undef LOG_IS_ON +#undef LOG_IF +#undef LOG_EVERY_N +#undef LOG_IF_EVERY_N +#undef DLOG +#undef DLOG_IF +#undef VLOG +#undef VLOG_IF +#undef VLOG_EVERY_N +#undef VLOG_IS_ON +#undef DVLOG +#undef LOG_FIRST_N +#undef CHECK +#undef DCHECK +#undef DCHECK_EQ +#undef PLOG +#undef PCHECK +#undef LOG_OCCURRENCES +#endif + +#define LOG_IS_ON(severity) \ + (severity >= libkineto::Logger::severityLevel()) + +#define LOG_IF(severity, condition) \ + !(LOG_IS_ON(severity) && (condition)) ? (void)0 : libkineto::VoidLogger() & \ + libkineto::Logger(severity, __LINE__, __FILE__).stream() + +#define LOG(severity) LOG_IF(severity, true) + +#define LOCAL_VARNAME_CONCAT(name, suffix) _##name##suffix##_ + +#define LOCAL_VARNAME(name) LOCAL_VARNAME_CONCAT(name, __LINE__) + +#define LOG_OCCURRENCES LOCAL_VARNAME(log_count) + +#define LOG_EVERY_N(severity, rate) \ + static int LOG_OCCURRENCES = 0; \ + LOG_IF(severity, LOG_OCCURRENCES++ % rate == 0) \ + << "(x" << LOG_OCCURRENCES << ") " + +template +struct __to_constant__ { + static const uint64_t val = n; +}; +#define FILENAME_HASH \ + __to_constant__::val +#define VLOG_IS_ON(verbosity) \ + (libkineto::Logger::verboseLogLevel() >= verbosity && \ + (libkineto::Logger::verboseLogModules() & FILENAME_HASH) == FILENAME_HASH) + +#define VLOG_IF(verbosity, condition) \ + LOG_IF(VERBOSE, VLOG_IS_ON(verbosity) && (condition)) + +#define VLOG(verbosity) VLOG_IF(verbosity, true) + +#define VLOG_EVERY_N(verbosity, rate) \ + static int LOG_OCCURRENCES = 0; \ + VLOG_IF(verbosity, LOG_OCCURRENCES++ % rate == 0) \ + << "(x" << LOG_OCCURRENCES << ") " + +#define PLOG(severity) \ + libkineto::Logger(severity, __LINE__, __FILE__, errno).stream() + +#define SET_LOG_SEVERITY_LEVEL(level) \ + libkineto::Logger::setSeverityLevel(level) + +#define SET_LOG_VERBOSITY_LEVEL(level, modules) \ + libkineto::Logger::setVerboseLogLevel(level); \ + libkineto::Logger::setVerboseLogModules(modules) + +// Logging the set of devices the trace is collect on. +#define LOGGER_OBSERVER_ADD_DEVICE(device_count) \ + libkineto::Logger::addLoggerObserverDevice(device_count) + +// Incrementing the number of events collected by this trace. +#define LOGGER_OBSERVER_ADD_EVENT_COUNT(count) \ + libkineto::Logger::addLoggerObserverEventCount(count) + +// Record duration of trace in milliseconds. +#define LOGGER_OBSERVER_SET_TRACE_DURATION_MS(duration) \ + libkineto::Logger::setLoggerObserverTraceDurationMS(duration) + +// Record the trace id when given. +#define LOGGER_OBSERVER_SET_TRACE_ID(tid) \ + libkineto::Logger::setLoggerObserverTraceID(tid) + +// Record the group trace id when given. +#define LOGGER_OBSERVER_SET_GROUP_TRACE_ID(gtid) \ + libkineto::Logger::setLoggerObserverGroupTraceID(gtid) + +// Log the set of destinations the trace is sent to. +#define LOGGER_OBSERVER_ADD_DESTINATION(dest) \ + libkineto::Logger::addLoggerObserverDestination(dest) + +// UST Logger Semantics to describe when a stage is complete. +#define UST_LOGGER_MARK_COMPLETED(stage) \ + LOG(libkineto::LoggerOutputType::STAGE) << "Completed Stage: " << stage + +#endif // USE_GOOGLE_LOG diff --git a/tb_plugins/profiling/libkineto/src/LoggerCollector.h b/tb_plugins/profiling/libkineto/src/LoggerCollector.h new file mode 100644 index 000000000..bb05aab21 --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/LoggerCollector.h @@ -0,0 +1,70 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#pragma once + +#if !USE_GOOGLE_LOG + +#include +#include +#include +#include + +// TODO(T90238193) +// @lint-ignore-every CLANGTIDY facebook-hte-RelativeInclude +#include "ILoggerObserver.h" + +namespace KINETO_NAMESPACE { + +using namespace libkineto; + +class LoggerCollector : public ILoggerObserver { + public: + LoggerCollector() : buckets_() {} + + void write(const std::string& message, LoggerOutputType ot = ERROR) override { + // Skip STAGE output type which is only used by USTLoggerCollector. + if (ot != STAGE) { + buckets_[ot].push_back(message); + } + } + + const std::map> extractCollectorMetadata() override { + return buckets_; + } + + void reset() override { + trace_duration_ms = 0; + event_count = 0; + destinations.clear(); + } + + void addDevice(const int64_t device) override { + devices.insert(device); + } + + void setTraceDurationMS(const int64_t duration) override { + trace_duration_ms = duration; + } + + void addEventCount(const int64_t count) override { + event_count += count; + } + + void addDestination(const std::string& dest) override { + destinations.insert(dest); + } + + protected: + std::map> buckets_; + + // These are useful metadata to collect from CUPTIActivityProfiler for internal tracking. + std::set devices; + int64_t trace_duration_ms{0}; + std::atomic event_count{0}; + std::set destinations; + +}; + +} // namespace KINETO_NAMESPACE + +#endif // !USE_GOOGLE_LOG diff --git a/tb_plugins/profiling/libkineto/src/RoctracerActivityApi.cpp b/tb_plugins/profiling/libkineto/src/RoctracerActivityApi.cpp new file mode 100644 index 000000000..73eff13e2 --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/RoctracerActivityApi.cpp @@ -0,0 +1,569 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#include "RoctracerActivityApi.h" + +#include +#include +#include + +#include "Demangle.h" +#include "output_base.h" +#include "ThreadUtil.h" + +typedef uint64_t timestamp_t; + +static timestamp_t timespec_to_ns(const timespec& time) { + return ((timestamp_t)time.tv_sec * 1000000000) + time.tv_nsec; + } + +using namespace std::chrono; + +namespace KINETO_NAMESPACE { + +constexpr size_t kBufSize(2 * 1024 * 1024); + +RoctracerActivityApi& RoctracerActivityApi::singleton() { + static RoctracerActivityApi instance; + return instance; +} + +RoctracerActivityApi::RoctracerActivityApi() { + gpuTraceBuffers_ = std::make_unique>(); +} + +RoctracerActivityApi::~RoctracerActivityApi() { + disableActivities(std::set()); + endTracing(); +} + +void RoctracerActivityApi::pushCorrelationID(int id, CorrelationFlowType type) { +#ifdef HAS_ROCTRACER + if (!singleton().externalCorrelationEnabled_) { + return; + } + // placeholder +#endif +} + +void RoctracerActivityApi::popCorrelationID(CorrelationFlowType type) { +#ifdef HAS_ROCTRACER + if (!singleton().externalCorrelationEnabled_) { + return; + } + // placeholder +#endif +} + +void RoctracerActivityApi::setMaxBufferSize(int size) { + maxGpuBufferCount_ = 1 + size / kBufSize; +} + +int RoctracerActivityApi::processActivities( + ActivityLogger& logger) { + // Find offset to map from monotonic clock to system clock. + // This will break time-ordering of events but is status quo. + + timespec t0, t1, t00; + clock_gettime(CLOCK_REALTIME, &t0); + clock_gettime(CLOCK_MONOTONIC, &t1); + clock_gettime(CLOCK_REALTIME, &t00); + + const timestamp_t toffset = (timespec_to_ns(t0) >> 1) + (timespec_to_ns(t00) >> 1) - timespec_to_ns(t1); + + int count = 0; + + // Basic Api calls + + for (auto &item : rows_) { + GenericTraceActivity a; + a.startTime = (item.begin + toffset) / 1000; + a.endTime = (item.end + toffset) / 1000; + a.id = item.id; + a.device = item.pid; + a.resource = item.tid; + a.activityType = ActivityType::CUDA_RUNTIME; + a.activityName = std::string(roctracer_op_string(ACTIVITY_DOMAIN_HIP_API, item.cid, 0)); + a.flow.id = item.id; + a.flow.type = kLinkAsyncCpuGpu; + a.flow.start = true; + + logger.handleGenericActivity(a); + ++count; + } + + // Malloc/Free calls + for (auto &item : mallocRows_) { + GenericTraceActivity a; + a.startTime = (item.begin + toffset) / 1000; + a.endTime = (item.end + toffset) / 1000; + a.id = item.id; + a.device = item.pid; + a.resource = item.tid; + a.activityType = ActivityType::CUDA_RUNTIME; + a.activityName = std::string(roctracer_op_string(ACTIVITY_DOMAIN_HIP_API, item.cid, 0)); + a.flow.id = item.id; + a.flow.type = kLinkAsyncCpuGpu; + a.flow.start = true; + + a.addMetadata("ptr", item.ptr); + if (item.cid == HIP_API_ID_hipMalloc) { + a.addMetadata("size", item.size); + } + + logger.handleGenericActivity(a); + ++count; + } + + // HipMemcpy calls + for (auto &item : copyRows_) { + GenericTraceActivity a; + a.startTime = (item.begin + toffset) / 1000; + a.endTime = (item.end + toffset) / 1000; + a.id = item.id; + a.device = item.pid; + a.resource = item.tid; + a.activityType = ActivityType::CUDA_RUNTIME; + a.activityName = std::string(roctracer_op_string(ACTIVITY_DOMAIN_HIP_API, item.cid, 0)); + a.flow.id = item.id; + a.flow.type = kLinkAsyncCpuGpu; + a.flow.start = true; + + a.addMetadata("src", item.src); + a.addMetadata("dst", item.dst); + a.addMetadata("size", item.size); + a.addMetadata("kind", item.kind); + if ((item.cid == HIP_API_ID_hipMemcpyAsync) || (item.cid == HIP_API_ID_hipMemcpyWithStream)) { + a.addMetadata("stream", fmt::format("{}", reinterpret_cast(item.stream))); + } + + logger.handleGenericActivity(a); + ++count; + } + + // Kernel Launch Api calls + + for (auto &item : kernelRows_) { + GenericTraceActivity a; + a.startTime = (item.begin + toffset) / 1000; + a.endTime = (item.end + toffset) / 1000; + a.id = item.id; + a.device = item.pid; + a.resource = item.tid; + a.activityType = ActivityType::CUDA_RUNTIME; + a.activityName = std::string(roctracer_op_string(ACTIVITY_DOMAIN_HIP_API, item.cid, 0)); + a.flow.id = item.id; + a.flow.type = kLinkAsyncCpuGpu; + a.flow.start = true; + + if (item.functionAddr != nullptr) { + a.addMetadataQuoted( + "kernel", demangle(hipKernelNameRefByPtr(item.functionAddr, item.stream))); + } + else if (item.function != nullptr) { + a.addMetadataQuoted( + "kernel", demangle(hipKernelNameRef(item.function))); + } + a.addMetadata("grid dim", fmt::format("[{}, {}, {}]", item.gridX, item.gridY, item.gridZ)); + a.addMetadata("block dim", fmt::format("[{}, {}, {}]", item.workgroupX, item.workgroupY, item.workgroupZ)); + a.addMetadata("shared size", item.groupSegmentSize); + a.addMetadata("stream", fmt::format("{}", reinterpret_cast(item.stream))); + + // Stash launches to tie to the async ops + kernelLaunches_[a.id] = a; + + // Stash kernel names to tie to the async ops + std::string name; + if (item.functionAddr != nullptr) { + name = demangle(hipKernelNameRefByPtr(item.functionAddr, item.stream)); + } + else if (item.function != nullptr) { + name = demangle(hipKernelNameRef(item.function)); + } + if (!name.empty()) { + uint32_t string_id = reverseStrings_[name]; + if (string_id == 0) { + string_id = nextStringId_++; + reverseStrings_[name] = string_id; + strings_[string_id] = name; + } + kernelNames_[item.id] = string_id; + } + + logger.handleGenericActivity(a); + ++count; + } + + // Async Ops + + for (auto& buffer : *gpuTraceBuffers_) { + const roctracer_record_t* record = (const roctracer_record_t*)(buffer.data); + const roctracer_record_t* end_record = (const roctracer_record_t*)(buffer.data + buffer.validSize); + GenericTraceActivity a; + + while (record < end_record) { + if ((record->domain == ACTIVITY_DOMAIN_HIP_API) && (loggedIds_.contains(record->op))) { + const char *name = roctracer_op_string(record->domain, record->op, record->kind); + a.device = record->process_id; + a.resource = record->thread_id; + + a.startTime = (record->begin_ns + toffset) / 1000; + a.endTime = (record->end_ns + toffset) / 1000; + a.id = record->correlation_id; + + a.activityType = ActivityType::CUDA_RUNTIME; + a.activityName = std::string(name); + a.flow.id = record->correlation_id; + a.flow.type = kLinkAsyncCpuGpu; + a.flow.start = true; + + logger.handleGenericActivity(a); + ++count; + } + else if (record->domain == ACTIVITY_DOMAIN_HCC_OPS) { + // Overlay launch metadata for kernels + auto kit = kernelLaunches_.find(record->correlation_id); + if (kit != kernelLaunches_.end()) { + a = (*kit).second; + } + + const char *name = roctracer_op_string(record->domain, record->op, record->kind); + a.device = record->device_id; + a.resource = record->queue_id; + + a.startTime = (record->begin_ns + toffset) / 1000; + a.endTime = (record->end_ns + toffset) / 1000; + a.id = record->correlation_id; + + a.activityType = ActivityType::CONCURRENT_KERNEL; + a.activityName = std::string(name); + a.flow.id = record->correlation_id; + a.flow.type = kLinkAsyncCpuGpu; + + auto it = kernelNames_.find(record->correlation_id); + if (it != kernelNames_.end()) { + a.activityName = strings_[it->second]; + } + + logger.handleGenericActivity(a); + ++count; + } + + roctracer_next_record(record, &record); + } + } + return count; +} + +void RoctracerActivityApi::clearActivities() { + gpuTraceBuffers_->clear(); + rows_.clear(); + kernelRows_.clear(); + copyRows_.clear(); + mallocRows_.clear(); + kernelLaunches_.clear(); +} + +void RoctracerActivityApi::api_callback(uint32_t domain, uint32_t cid, const void* callback_data, void* arg) +{ + RoctracerActivityApi *dis = &singleton(); + + if (domain == ACTIVITY_DOMAIN_HIP_API && dis->loggedIds_.contains(cid)) { + const hip_api_data_t* data = (const hip_api_data_t*)(callback_data); + + // Pack callbacks into row structures + + static timespec timestamp; // FIXME verify thread safety + + if (data->phase == ACTIVITY_API_PHASE_ENTER) { + clock_gettime(CLOCK_MONOTONIC, ×tamp); // record proper clock + } + else { // (data->phase == ACTIVITY_API_PHASE_EXIT) + timespec endTime; + timespec startTime { timestamp }; + clock_gettime(CLOCK_MONOTONIC, &endTime); // record proper clock + + switch (cid) { + case HIP_API_ID_hipLaunchKernel: + case HIP_API_ID_hipExtLaunchKernel: + case HIP_API_ID_hipLaunchCooperativeKernel: // Should work here + { + auto &args = data->args.hipLaunchKernel; + dis->kernelRows_.emplace_back(data->correlation_id, + domain, + cid, + processId(), + systemThreadId(), + timespec_to_ns(startTime), + timespec_to_ns(endTime), + args.function_address, + nullptr, + args.numBlocks.x, + args.numBlocks.y, + args.numBlocks.z, + args.dimBlocks.x, + args.dimBlocks.y, + args.dimBlocks.z, + args.sharedMemBytes, + args.stream + ); + } + break; + case HIP_API_ID_hipHccModuleLaunchKernel: + case HIP_API_ID_hipModuleLaunchKernel: + case HIP_API_ID_hipExtModuleLaunchKernel: + { + auto &args = data->args.hipModuleLaunchKernel; + dis->kernelRows_.emplace_back(data->correlation_id, + domain, + cid, + processId(), + systemThreadId(), + timespec_to_ns(startTime), + timespec_to_ns(endTime), + nullptr, + args.f, + args.gridDimX, + args.gridDimY, + args.gridDimZ, + args.blockDimX, + args.blockDimY, + args.blockDimZ, + args.sharedMemBytes, + args.stream + ); + } + break; + case HIP_API_ID_hipLaunchCooperativeKernelMultiDevice: + case HIP_API_ID_hipExtLaunchMultiKernelMultiDevice: +#if 0 + { + auto &args = data->args.hipLaunchCooperativeKernelMultiDevice.launchParamsList__val; + dis->kernelRows_.emplace_back(data->correlation_id, + domain, + cid, + processId(), + systemThreadId(), + timespec_to_ns(startTime), + timespec_to_ns(endTime), + args.function_address, + nullptr, + args.numBlocks.x, + args.numBlocks.y, + args.numBlocks.z, + args.dimBlocks.x, + args.dimBlocks.y, + args.dimBlocks.z, + args.sharedMemBytes, + args.stream + ); + } +#endif + break; + case HIP_API_ID_hipMalloc: + dis->mallocRows_.emplace_back(data->correlation_id, + domain, + cid, + processId(), + systemThreadId(), + timespec_to_ns(startTime), + timespec_to_ns(endTime), + data->args.hipMalloc.ptr__val, + data->args.hipMalloc.size + ); + break; + case HIP_API_ID_hipFree: + dis->mallocRows_.emplace_back(data->correlation_id, + domain, + cid, + processId(), + systemThreadId(), + timespec_to_ns(startTime), + timespec_to_ns(endTime), + data->args.hipFree.ptr, + 0 + ); + break; + case HIP_API_ID_hipMemcpy: + { + auto &args = data->args.hipMemcpy; + dis->copyRows_.emplace_back(data->correlation_id, + domain, + cid, + processId(), + systemThreadId(), + timespec_to_ns(startTime), + timespec_to_ns(endTime), + args.src, + args.dst, + args.sizeBytes, + args.kind, + static_cast(0) // use placeholder? + ); + } + break; + case HIP_API_ID_hipMemcpyAsync: + case HIP_API_ID_hipMemcpyWithStream: + { + auto &args = data->args.hipMemcpyAsync; + dis->copyRows_.emplace_back(data->correlation_id, + domain, + cid, + processId(), + systemThreadId(), + timespec_to_ns(startTime), + timespec_to_ns(endTime), + args.src, + args.dst, + args.sizeBytes, + args.kind, + args.stream + ); + } + break; + default: + dis->rows_.emplace_back(data->correlation_id, + domain, + cid, + processId(), + systemThreadId(), + timespec_to_ns(startTime), + timespec_to_ns(endTime) + ); + break; + } + } + } +} + +void RoctracerActivityApi::activity_callback(const char* begin, const char* end, void* arg) +{ + size_t size = end - begin; + uint8_t *buffer = (uint8_t*) malloc(size); + auto &gpuTraceBuffers = singleton().gpuTraceBuffers_; + memcpy(buffer, begin, size); + gpuTraceBuffers->emplace_back(buffer, size); +} + +void RoctracerActivityApi::enableActivities( + const std::set& selected_activities) { +#ifdef HAS_ROCTRACER + if (!registered_) { + roctracer_set_properties(ACTIVITY_DOMAIN_HIP_API, nullptr); // Magic encantation + + // Set some api calls to ignore + loggedIds_.setInvertMode(true); // Omit the specified api + loggedIds_.add("hipGetDevice"); + loggedIds_.add("hipSetDevice"); + loggedIds_.add("hipGetLastError"); + loggedIds_.add("__hipPushCallConfiguration"); + loggedIds_.add("__hipPopCallConfiguration"); + loggedIds_.add("hipCtxSetCurrent"); + loggedIds_.add("hipEventRecord"); + loggedIds_.add("hipEventQuery"); + loggedIds_.add("hipGetDeviceProperties"); + loggedIds_.add("hipPeekAtLastError"); + loggedIds_.add("hipModuleGetFunction"); + loggedIds_.add("hipEventCreateWithFlags"); + + // Enable API callbacks + if (loggedIds_.invertMode() == true) { + // exclusion list - enable entire domain and turn off things in list + roctracer_enable_domain_callback(ACTIVITY_DOMAIN_HIP_API, api_callback, nullptr); + const std::unordered_map &filter = loggedIds_.filterList(); + for (auto it = filter.begin(); it != filter.end(); ++it) { + roctracer_disable_op_callback(ACTIVITY_DOMAIN_HIP_API, it->first); + } + } + else { + // inclusion list - only enable things in the list + const std::unordered_map &filter = loggedIds_.filterList(); + roctracer_disable_domain_callback(ACTIVITY_DOMAIN_HIP_API); + for (auto it = filter.begin(); it != filter.end(); ++it) { + roctracer_enable_op_callback(ACTIVITY_DOMAIN_HIP_API, it->first, api_callback, nullptr); + } + } + //roctracer_enable_domain_callback(ACTIVITY_DOMAIN_ROCTX, api_callback, nullptr); + + // Allocate default tracing pool + roctracer_properties_t properties; + memset(&properties, 0, sizeof(roctracer_properties_t)); + properties.buffer_size = 0x1000; + roctracer_open_pool(&properties); + + // Enable async op collection + roctracer_properties_t hcc_cb_properties; + memset(&hcc_cb_properties, 0, sizeof(roctracer_properties_t)); + hcc_cb_properties.buffer_size = 0x4000; + hcc_cb_properties.buffer_callback_fun = activity_callback; + roctracer_open_pool_expl(&hcc_cb_properties, &hccPool_); + roctracer_enable_domain_activity_expl(ACTIVITY_DOMAIN_HCC_OPS, hccPool_); + + registered_ = true; + } + + for (const auto& activity : selected_activities) { + if (activity == ActivityType::EXTERNAL_CORRELATION) { + externalCorrelationEnabled_ = true; + } + } + + roctracer_start(); +#endif +} + +void RoctracerActivityApi::disableActivities( + const std::set& selected_activities) { +#ifdef HAS_ROCTRACER + roctracer_stop(); + roctracer_flush_activity_expl(hccPool_); + + for (const auto& activity : selected_activities) { + if (activity == ActivityType::EXTERNAL_CORRELATION) { + externalCorrelationEnabled_ = false; + } + } +#endif +} + +void RoctracerActivityApi::endTracing() { + if (registered_ == true) { + roctracer_disable_domain_callback(ACTIVITY_DOMAIN_HIP_API); + //roctracer_disable_domain_callback(ACTIVITY_DOMAIN_ROCTX); + + roctracer_disable_domain_activity(ACTIVITY_DOMAIN_HCC_OPS); + roctracer_close_pool_expl(hccPool_); + } +} + + +ApiIdList::ApiIdList() +: invert_(true) +{ +} + +void ApiIdList::add(std::string apiName) +{ + uint32_t cid = 0; + if (roctracer_op_code(ACTIVITY_DOMAIN_HIP_API, apiName.c_str(), &cid, nullptr) == ROCTRACER_STATUS_SUCCESS) { + filter_[cid] = 1; + } +} +void ApiIdList::remove(std::string apiName) +{ + uint32_t cid = 0; + if (roctracer_op_code(ACTIVITY_DOMAIN_HIP_API, apiName.c_str(), &cid, nullptr) == ROCTRACER_STATUS_SUCCESS) { + filter_.erase(cid); + } +} + +bool ApiIdList::loadUserPrefs() +{ + // placeholder + return false; +} +bool ApiIdList::contains(uint32_t apiId) +{ + return (filter_.find(apiId) != filter_.end()) ? !invert_ : invert_; // XOR +} + +} // namespace KINETO_NAMESPACE diff --git a/tb_plugins/profiling/libkineto/src/RoctracerActivityApi.h b/tb_plugins/profiling/libkineto/src/RoctracerActivityApi.h new file mode 100644 index 000000000..28280253e --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/RoctracerActivityApi.h @@ -0,0 +1,171 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef HAS_ROCTRACER +#include +#include +#include +#include +#include +#endif + +#include "ActivityType.h" +#include "GenericTraceActivity.h" +#include "RoctracerActivityBuffer.h" + + +namespace KINETO_NAMESPACE { + +using namespace libkineto; + +class ApiIdList +{ +public: + ApiIdList(); + bool invertMode() { return invert_; } + void setInvertMode(bool invert) { invert_ = invert; } + void add(std::string apiName); + void remove(std::string apiName); + bool loadUserPrefs(); + bool contains(uint32_t apiId); + const std::unordered_map &filterList() { return filter_; } + +private: + std::unordered_map filter_; + bool invert_; +}; + +struct roctracerRow { + roctracerRow(uint64_t id, uint32_t domain, uint32_t cid, uint32_t pid + , uint32_t tid, uint64_t begin, uint64_t end) + : id(id), domain(domain), cid(cid), pid(pid), tid(tid), begin(begin), end(end) {} + uint64_t id; // correlation_id + uint32_t domain; + uint32_t cid; + uint32_t pid; + uint32_t tid; + uint64_t begin; + uint64_t end; +}; + +struct kernelRow : public roctracerRow { + kernelRow(uint64_t id, uint32_t domain, uint32_t cid, uint32_t pid + , uint32_t tid, uint64_t begin, uint64_t end + , const void *faddr, hipFunction_t function + , unsigned int gx, unsigned int gy, unsigned int gz + , unsigned int wx, unsigned int wy, unsigned int wz + , size_t gss, hipStream_t stream) + : roctracerRow(id, domain, cid, pid, tid, begin, end), functionAddr(faddr) + , function(function), gridX(gx), gridY(gy), gridZ(gz) + , workgroupX(wx), workgroupY(wy), workgroupZ(wz), groupSegmentSize(gss) + , stream(stream) {} + const void* functionAddr; + hipFunction_t function; + unsigned int gridX; + unsigned int gridY; + unsigned int gridZ; + unsigned int workgroupX; + unsigned int workgroupY; + unsigned int workgroupZ; + size_t groupSegmentSize; + hipStream_t stream; +}; + +struct copyRow : public roctracerRow { + copyRow(uint64_t id, uint32_t domain, uint32_t cid, uint32_t pid + , uint32_t tid, uint64_t begin, uint64_t end + , const void* src, const void *dst, size_t size, hipMemcpyKind kind + , hipStream_t stream) + : roctracerRow(id, domain, cid, pid, tid, begin, end) + , src(src), dst(dst), size(size), kind(kind), stream(stream) {} + const void *src; + const void *dst; + size_t size; + hipMemcpyKind kind; + hipStream_t stream; +}; + +struct mallocRow : public roctracerRow { + mallocRow(uint64_t id, uint32_t domain, uint32_t cid, uint32_t pid + , uint32_t tid, uint64_t begin, uint64_t end + , const void* ptr, size_t size) + : roctracerRow(id, domain, cid, pid, tid, begin, end) + , ptr(ptr), size(size) {} + const void *ptr; + size_t size; +}; + + +class RoctracerActivityApi { + public: + enum CorrelationFlowType { + Default, + User + }; + + RoctracerActivityApi(); + RoctracerActivityApi(const RoctracerActivityApi&) = delete; + RoctracerActivityApi& operator=(const RoctracerActivityApi&) = delete; + + virtual ~RoctracerActivityApi(); + + static RoctracerActivityApi& singleton(); + + static void pushCorrelationID(int id, CorrelationFlowType type); + static void popCorrelationID(CorrelationFlowType type); + + void enableActivities( + const std::set& selected_activities); + void disableActivities( + const std::set& selected_activities); + void clearActivities(); + + int processActivities(ActivityLogger& logger); + + void setMaxBufferSize(int size); + + std::atomic_bool stopCollection{false}; + + private: + bool registered_{false}; + void endTracing(); + +#ifdef HAS_ROCTRACER + roctracer_pool_t *hccPool_{NULL}; + static void api_callback(uint32_t domain, uint32_t cid, const void* callback_data, void* arg); + static void activity_callback(const char* begin, const char* end, void* arg); + + //Name cache + uint32_t nextStringId_{2}; + std::map strings_; + std::map reverseStrings_; + std::map kernelNames_; + + ApiIdList loggedIds_; + + // Api callback data + std::deque rows_; + std::deque kernelRows_; + std::deque copyRows_; + std::deque mallocRows_; + std::map kernelLaunches_; +#endif + + int maxGpuBufferCount_{0}; + std::unique_ptr> gpuTraceBuffers_; + bool externalCorrelationEnabled_{true}; +}; + +} // namespace KINETO_NAMESPACE + diff --git a/tb_plugins/profiling/libkineto/src/RoctracerActivityBuffer.h b/tb_plugins/profiling/libkineto/src/RoctracerActivityBuffer.h new file mode 100644 index 000000000..cd8a5709a --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/RoctracerActivityBuffer.h @@ -0,0 +1,30 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#pragma once + +#include +#include +#include +#include + +namespace KINETO_NAMESPACE { + +class RoctracerActivityBuffer { + public: + // data must be allocated using malloc. + // Ownership is transferred to this object. + RoctracerActivityBuffer(uint8_t* data, size_t validSize) + : data(data), validSize(validSize) {} + + ~RoctracerActivityBuffer() { + free(data); + } + + // Allocated by malloc + uint8_t* data{nullptr}; + + // Number of bytes used + size_t validSize; +}; + +} // namespace KINETO_NAMESPACE diff --git a/tb_plugins/profiling/libkineto/src/SampleListener.h b/tb_plugins/profiling/libkineto/src/SampleListener.h new file mode 100644 index 000000000..bff86ad12 --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/SampleListener.h @@ -0,0 +1,146 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#pragma once + +#include +#include +#include +#include +#include + +namespace KINETO_NAMESPACE { + +class Config; + +class SampleValue { + public: + template + explicit SampleValue(T v) { + init(v); + } + + SampleValue(const SampleValue&) = default; + SampleValue& operator=(const SampleValue&) = delete; + SampleValue(SampleValue&&) = default; + SampleValue& operator=(SampleValue&&) = default; + + bool isInt() const { + return type_ == INT64; + } + + int64_t getInt() const { + assert(isInt()); + return int_; + } + + bool isDouble() const { + return type_ == DOUBLE; + } + + double getDouble() const { + assert(isDouble()); + return dbl_; + } + + inline void operator*=(double x) { + assert(isDouble() || isInt()); + if (isDouble()) { + dbl_ *= x; + } else { + int_ = std::round(int_ * x); + } + } + + inline bool operator<(const SampleValue& o) const { + if (type_ != o.type_) { + return type_ < o.type_; + } else if (type_ == INT64) { + return int_ < o.int_; + } else if (type_ == DOUBLE) { + return dbl_ < o.dbl_; + } + assert(false); + return true; + } + + void print(std::ostream& s) const { + if (type_ == INT64) { + s << int_; + } else if (type_ == DOUBLE) { + s << dbl_; + } else { + assert(false); + } + } + + private: + enum Type { INT64, DOUBLE }; + + template + void init(T v); + + Type type_{INT64}; + union { + int64_t int_{0}; + double dbl_; + }; +}; + +template <> +inline void SampleValue::init(uint64_t v) { + int_ = v, type_ = INT64; +} +template <> +inline void SampleValue::init(int64_t v) { + int_ = v, type_ = INT64; +} +template <> +inline void SampleValue::init(int v) { + int_ = v, type_ = INT64; +} +template <> +inline void SampleValue::init(double v) { + dbl_ = v, type_ = DOUBLE; +} + +inline std::ostream& operator<<(std::ostream& out, const SampleValue& s) { + s.print(out); + return out; +} + +using PercentileList = std::vector>; + +struct Stat { + const std::string& name; + const PercentileList percentileValues; + SampleValue total; +}; + +struct Sample { + Sample(int stats_count) { + stats.reserve(stats_count); + } + + // Offset in milliseconds from first sample in report + int deltaMsec; + std::vector stats; +}; + +// Inherit from this to be notified of samples +class SampleListener { + public: + SampleListener(const SampleListener&) = delete; + SampleListener& operator=(const SampleListener&) = delete; + + virtual ~SampleListener(){}; + + // Report bucketed & aggregated values for event + virtual void handleSample(int device, const Sample& sample, bool from_new_version) = 0; + + virtual void update(const Config& config) = 0; + + protected: + SampleListener() = default; +}; + +} // namespace KINETO_NAMESPACE diff --git a/tb_plugins/profiling/libkineto/src/ScopeExit.h b/tb_plugins/profiling/libkineto/src/ScopeExit.h new file mode 100644 index 000000000..b9a6bc83e --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/ScopeExit.h @@ -0,0 +1,29 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#pragma once + +// Implement a simple scope handler allowing a function to release +// resources when an error or exception occurs + +template +class ScopeExit { + public: + explicit ScopeExit(T t) : t(t) {} + ~ScopeExit() { + t(); + } + T t; +}; + +template +ScopeExit makeScopeExit(T t) { + return ScopeExit(t); +}; + +// Add a level of indirection so __LINE__ is expanded +#define __kINETO_CONCAT(name, line) name##line +#define ANON_VAR(name, line) __kINETO_CONCAT(name, line) + +#define SCOPE_EXIT(func) \ + const auto ANON_VAR(SCOPE_BLOCK, __LINE__) = \ + makeScopeExit([=]() { func; }) diff --git a/tb_plugins/profiling/libkineto/src/ThreadUtil.cpp b/tb_plugins/profiling/libkineto/src/ThreadUtil.cpp new file mode 100644 index 000000000..0f67d54d5 --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/ThreadUtil.cpp @@ -0,0 +1,203 @@ +#include "ThreadUtil.h" + +#ifndef _MSC_VER +#include +#include +#include +#include +#else // _MSC_VER +#include +#include +#define WIN32_LEAN_AND_MEAN +#define NOGDI +#include +#include +#undef ERROR +#endif // _MSC_VER + +#ifdef __ANDROID__ +#include +#endif + +#include +#include +#include + +namespace libkineto { + +namespace { +thread_local int32_t _pid = 0; +thread_local int32_t _tid = 0; +thread_local int32_t _sysTid = 0; +} + +int32_t processId() { + if (!_pid) { +#ifndef _MSC_VER + _pid = (int32_t)getpid(); +#else + _pid = (int32_t)GetCurrentProcessId(); +#endif + } + return _pid; +} + +int32_t systemThreadId() { + if (!_sysTid) { +#ifdef __APPLE__ + _sysTid = (int32_t)syscall(SYS_thread_selfid); +#elif defined _MSC_VER + _sysTid = (int32_t)GetCurrentThreadId(); +#else + _sysTid = (int32_t)syscall(SYS_gettid); +#endif + } + return _sysTid; +} + +int32_t threadId() { + if (!_tid) { +#ifdef __APPLE__ + uint64_t tid; + pthread_threadid_np(nullptr, &tid); + _tid = tid; +#elif defined _MSC_VER + _tid = (int32_t)GetCurrentThreadId(); +#else + pthread_t pth = pthread_self(); + int32_t* ptr = reinterpret_cast(&pth); + _tid = *ptr; +#endif + } + return _tid; +} + +namespace { +static constexpr size_t kMaxThreadNameLength = 16; + +static constexpr const char* basename(const char* s, int off = 0) { + return !s[off] + ? s + : s[off] == '/' ? basename(&s[off + 1]) : basename(s, off + 1); +} +#if defined(_MSC_VER) +void *getKernel32Func(const char* procName) { + return GetProcAddress(GetModuleHandleA("KERNEL32.DLL"), procName); +} +#endif +} + +bool setThreadName(const std::string& name) { +#ifdef __APPLE__ + return 0 == pthread_setname_np(name.c_str()); +#elif defined _MSC_VER + // Per https://docs.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-setthreaddescription + // Use runtime linking to set thread description + static auto _SetThreadDescription = reinterpret_cast(getKernel32Func("SetThreadDescription")); + if (!_SetThreadDescription) { + return false; + } + std::wstring_convert> conv; + std::wstring wname = conv.from_bytes(name); + HRESULT hr = _SetThreadDescription(GetCurrentThread(), wname.c_str()); + return SUCCEEDED(hr); +#else + return 0 == pthread_setname_np(pthread_self(), name.c_str()); +#endif +} + +std::string getThreadName() { +#ifndef _MSC_VER + char buf[kMaxThreadNameLength] = ""; + if ( +#ifndef __ANDROID__ + pthread_getname_np(pthread_self(), buf, kMaxThreadNameLength) != 0 +#else + prctl(PR_GET_NAME, buf, kMaxThreadNameLength) != 0 +#endif + ) { + return "Unknown"; + } + return buf; +#else // _MSC_VER + static auto _GetThreadDescription = reinterpret_cast(getKernel32Func("GetThreadDescription")); + if (!_GetThreadDescription) { + return "Unknown"; + } + PWSTR data; + HRESULT hr = _GetThreadDescription(GetCurrentThread(), &data); + if (!SUCCEEDED(hr)) { + return ""; + } + std::wstring_convert> conv; + std::string name = conv.to_bytes(data); + LocalFree(data); + return name; +#endif +} + +// Linux: +// Extract process name from /proc/pid/cmdline. This does not have +// the 16 character limit that /proc/pid/status and /prod/pid/comm has. +std::string processName(int32_t pid) { +#ifdef __linux__ + FILE* cmdfile = fopen(fmt::format("/proc/{}/cmdline", pid).c_str(), "r"); + if (cmdfile != nullptr) { + char* command = nullptr; + int scanned = fscanf(cmdfile, "%ms", &command); + fclose(cmdfile); + if (scanned > 0 && command) { + std::string ret(basename(command)); + free(command); + return ret; + } + } + std::cerr << "Failed to read process name for pid " << pid << std::endl; +#endif + return ""; +} + +// Max number of parent pids to collect, just for extra safeguarding. +constexpr int kMaxParentPids = 10; + +// Return a pair of +static std::pair parentPidAndCommand(int32_t pid) { +#ifdef __linux__ + FILE* statfile = fopen(fmt::format("/proc/{}/stat", pid).c_str(), "r"); + if (statfile == nullptr) { + return std::make_pair(0, ""); + } + int32_t parent_pid; + char* command = nullptr; + int scanned = fscanf(statfile, "%*d (%m[^)]) %*c %d", &command, &parent_pid); + fclose(statfile); + std::pair ret; + if (scanned == 2) { + ret = std::make_pair(parent_pid, std::string(command)); + } else { + std::cerr << "Failed to parse /proc/" << pid << "/stat" << std::endl; + ret = std::make_pair(0, ""); + } + + // The 'm' character in the format tells fscanf to allocate memory + // for the parsed string, which we need to free here. + free(command); + return ret; +#else + return std::make_pair(0, ""); +#endif +} + +std::vector> pidCommandPairsOfAncestors() { + std::vector> pairs; + pairs.reserve(kMaxParentPids + 1); + int32_t curr_pid = processId(); + for (int i = 0; i <= kMaxParentPids && curr_pid > 1; i++) { + std::pair ppid_and_comm = parentPidAndCommand(curr_pid); + pairs.push_back(std::make_pair(curr_pid, ppid_and_comm.second)); + curr_pid = ppid_and_comm.first; + } + return pairs; +} + +} // namespace libkineto diff --git a/tb_plugins/profiling/libkineto/src/WeakSymbols.cpp b/tb_plugins/profiling/libkineto/src/WeakSymbols.cpp new file mode 100644 index 000000000..540a5ac8f --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/WeakSymbols.cpp @@ -0,0 +1,12 @@ +#include + +#ifndef _MSC_VER +extern "C" { +// This function is needed to avoid superfluous dependency on GNU OpenMP library when cuPTI is linked statically +// For more details see https://github.com/pytorch/pytorch/issues/51026 +__attribute__((weak)) int acc_get_device_type() { + throw std::runtime_error("Dummy implementation of acc_get_device_type is not supposed to be called!"); +} + +} // extern "C" +#endif diff --git a/tb_plugins/profiling/libkineto/src/cupti_call.h b/tb_plugins/profiling/libkineto/src/cupti_call.h new file mode 100644 index 000000000..fd6ebae76 --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/cupti_call.h @@ -0,0 +1,33 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#pragma once + +#include + +#ifdef HAS_CUPTI + +#include + +#define CUPTI_CALL(call) \ + [&]() -> CUptiResult { \ + CUptiResult _status_ = call; \ + if (_status_ != CUPTI_SUCCESS) { \ + const char* _errstr_ = nullptr; \ + cuptiGetResultString(_status_, &_errstr_); \ + LOG(WARNING) << fmt::format( \ + "function {} failed with error {} ({})", \ + #call, \ + _errstr_, \ + (int)_status_); \ + } \ + return _status_; \ + }() + +#define CUPTI_CALL_NOWARN(call) call + +#else + +#define CUPTI_CALL(call) call +#define CUPTI_CALL_NOWARN(call) call + +#endif // HAS_CUPTI diff --git a/tb_plugins/profiling/libkineto/src/cupti_strings.cpp b/tb_plugins/profiling/libkineto/src/cupti_strings.cpp new file mode 100644 index 000000000..4535273a2 --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/cupti_strings.cpp @@ -0,0 +1,502 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#include "cupti_strings.h" + +namespace libkineto { + +const char* memcpyKindString( + CUpti_ActivityMemcpyKind kind) { + switch (kind) { + case CUPTI_ACTIVITY_MEMCPY_KIND_HTOD: + return "HtoD"; + case CUPTI_ACTIVITY_MEMCPY_KIND_DTOH: + return "DtoH"; + case CUPTI_ACTIVITY_MEMCPY_KIND_HTOA: + return "HtoA"; + case CUPTI_ACTIVITY_MEMCPY_KIND_ATOH: + return "AtoH"; + case CUPTI_ACTIVITY_MEMCPY_KIND_ATOA: + return "AtoA"; + case CUPTI_ACTIVITY_MEMCPY_KIND_ATOD: + return "AtoD"; + case CUPTI_ACTIVITY_MEMCPY_KIND_DTOA: + return "DtoA"; + case CUPTI_ACTIVITY_MEMCPY_KIND_DTOD: + return "DtoD"; + case CUPTI_ACTIVITY_MEMCPY_KIND_HTOH: + return "HtoH"; + case CUPTI_ACTIVITY_MEMCPY_KIND_PTOP: + return "PtoP"; + default: + break; + } + return ""; +} + +const char* memoryKindString( + CUpti_ActivityMemoryKind kind) { + switch (kind) { + case CUPTI_ACTIVITY_MEMORY_KIND_UNKNOWN: + return "Unknown"; + case CUPTI_ACTIVITY_MEMORY_KIND_PAGEABLE: + return "Pageable"; + case CUPTI_ACTIVITY_MEMORY_KIND_PINNED: + return "Pinned"; + case CUPTI_ACTIVITY_MEMORY_KIND_DEVICE: + return "Device"; + case CUPTI_ACTIVITY_MEMORY_KIND_ARRAY: + return "Array"; + case CUPTI_ACTIVITY_MEMORY_KIND_MANAGED: + return "Managed"; + case CUPTI_ACTIVITY_MEMORY_KIND_DEVICE_STATIC: + return "Device Static"; + case CUPTI_ACTIVITY_MEMORY_KIND_MANAGED_STATIC: + return "Managed Static"; + case CUPTI_ACTIVITY_MEMORY_KIND_FORCE_INT: + return "Force Int"; + default: + return "Unrecognized"; + } +} + +const char* overheadKindString( + CUpti_ActivityOverheadKind kind) { + switch (kind) { + case CUPTI_ACTIVITY_OVERHEAD_UNKNOWN: + return "Unknown"; + case CUPTI_ACTIVITY_OVERHEAD_DRIVER_COMPILER: + return "Driver Compiler"; + case CUPTI_ACTIVITY_OVERHEAD_CUPTI_BUFFER_FLUSH: + return "Buffer Flush"; + case CUPTI_ACTIVITY_OVERHEAD_CUPTI_INSTRUMENTATION: + return "Instrumentation"; + case CUPTI_ACTIVITY_OVERHEAD_CUPTI_RESOURCE: + return "Resource"; + case CUPTI_ACTIVITY_OVERHEAD_FORCE_INT: + return "Force Int"; + default: + return "Unrecognized"; + } +} + + + +static const char* runtimeCbidNames[] = { + "INVALID", + "cudaDriverGetVersion", + "cudaRuntimeGetVersion", + "cudaGetDeviceCount", + "cudaGetDeviceProperties", + "cudaChooseDevice", + "cudaGetChannelDesc", + "cudaCreateChannelDesc", + "cudaConfigureCall", + "cudaSetupArgument", + "cudaGetLastError", + "cudaPeekAtLastError", + "cudaGetErrorString", + "cudaLaunch", + "cudaFuncSetCacheConfig", + "cudaFuncGetAttributes", + "cudaSetDevice", + "cudaGetDevice", + "cudaSetValidDevices", + "cudaSetDeviceFlags", + "cudaMalloc", + "cudaMallocPitch", + "cudaFree", + "cudaMallocArray", + "cudaFreeArray", + "cudaMallocHost", + "cudaFreeHost", + "cudaHostAlloc", + "cudaHostGetDevicePointer", + "cudaHostGetFlags", + "cudaMemGetInfo", + "cudaMemcpy", + "cudaMemcpy2D", + "cudaMemcpyToArray", + "cudaMemcpy2DToArray", + "cudaMemcpyFromArray", + "cudaMemcpy2DFromArray", + "cudaMemcpyArrayToArray", + "cudaMemcpy2DArrayToArray", + "cudaMemcpyToSymbol", + "cudaMemcpyFromSymbol", + "cudaMemcpyAsync", + "cudaMemcpyToArrayAsync", + "cudaMemcpyFromArrayAsync", + "cudaMemcpy2DAsync", + "cudaMemcpy2DToArrayAsync", + "cudaMemcpy2DFromArrayAsync", + "cudaMemcpyToSymbolAsync", + "cudaMemcpyFromSymbolAsync", + "cudaMemset", + "cudaMemset2D", + "cudaMemsetAsync", + "cudaMemset2DAsync", + "cudaGetSymbolAddress", + "cudaGetSymbolSize", + "cudaBindTexture", + "cudaBindTexture2D", + "cudaBindTextureToArray", + "cudaUnbindTexture", + "cudaGetTextureAlignmentOffset", + "cudaGetTextureReference", + "cudaBindSurfaceToArray", + "cudaGetSurfaceReference", + "cudaGLSetGLDevice", + "cudaGLRegisterBufferObject", + "cudaGLMapBufferObject", + "cudaGLUnmapBufferObject", + "cudaGLUnregisterBufferObject", + "cudaGLSetBufferObjectMapFlags", + "cudaGLMapBufferObjectAsync", + "cudaGLUnmapBufferObjectAsync", + "cudaWGLGetDevice", + "cudaGraphicsGLRegisterImage", + "cudaGraphicsGLRegisterBuffer", + "cudaGraphicsUnregisterResource", + "cudaGraphicsResourceSetMapFlags", + "cudaGraphicsMapResources", + "cudaGraphicsUnmapResources", + "cudaGraphicsResourceGetMappedPointer", + "cudaGraphicsSubResourceGetMappedArray", + "cudaVDPAUGetDevice", + "cudaVDPAUSetVDPAUDevice", + "cudaGraphicsVDPAURegisterVideoSurface", + "cudaGraphicsVDPAURegisterOutputSurface", + "cudaD3D11GetDevice", + "cudaD3D11GetDevices", + "cudaD3D11SetDirect3DDevice", + "cudaGraphicsD3D11RegisterResource", + "cudaD3D10GetDevice", + "cudaD3D10GetDevices", + "cudaD3D10SetDirect3DDevice", + "cudaGraphicsD3D10RegisterResource", + "cudaD3D10RegisterResource", + "cudaD3D10UnregisterResource", + "cudaD3D10MapResources", + "cudaD3D10UnmapResources", + "cudaD3D10ResourceSetMapFlags", + "cudaD3D10ResourceGetSurfaceDimensions", + "cudaD3D10ResourceGetMappedArray", + "cudaD3D10ResourceGetMappedPointer", + "cudaD3D10ResourceGetMappedSize", + "cudaD3D10ResourceGetMappedPitch", + "cudaD3D9GetDevice", + "cudaD3D9GetDevices", + "cudaD3D9SetDirect3DDevice", + "cudaD3D9GetDirect3DDevice", + "cudaGraphicsD3D9RegisterResource", + "cudaD3D9RegisterResource", + "cudaD3D9UnregisterResource", + "cudaD3D9MapResources", + "cudaD3D9UnmapResources", + "cudaD3D9ResourceSetMapFlags", + "cudaD3D9ResourceGetSurfaceDimensions", + "cudaD3D9ResourceGetMappedArray", + "cudaD3D9ResourceGetMappedPointer", + "cudaD3D9ResourceGetMappedSize", + "cudaD3D9ResourceGetMappedPitch", + "cudaD3D9Begin", + "cudaD3D9End", + "cudaD3D9RegisterVertexBuffer", + "cudaD3D9UnregisterVertexBuffer", + "cudaD3D9MapVertexBuffer", + "cudaD3D9UnmapVertexBuffer", + "cudaThreadExit", + "cudaSetDoubleForDevice", + "cudaSetDoubleForHost", + "cudaThreadSynchronize", + "cudaThreadGetLimit", + "cudaThreadSetLimit", + "cudaStreamCreate", + "cudaStreamDestroy", + "cudaStreamSynchronize", + "cudaStreamQuery", + "cudaEventCreate", + "cudaEventCreateWithFlags", + "cudaEventRecord", + "cudaEventDestroy", + "cudaEventSynchronize", + "cudaEventQuery", + "cudaEventElapsedTime", + "cudaMalloc3D", + "cudaMalloc3DArray", + "cudaMemset3D", + "cudaMemset3DAsync", + "cudaMemcpy3D", + "cudaMemcpy3DAsync", + "cudaThreadSetCacheConfig", + "cudaStreamWaitEvent", + "cudaD3D11GetDirect3DDevice", + "cudaD3D10GetDirect3DDevice", + "cudaThreadGetCacheConfig", + "cudaPointerGetAttributes", + "cudaHostRegister", + "cudaHostUnregister", + "cudaDeviceCanAccessPeer", + "cudaDeviceEnablePeerAccess", + "cudaDeviceDisablePeerAccess", + "cudaPeerRegister", + "cudaPeerUnregister", + "cudaPeerGetDevicePointer", + "cudaMemcpyPeer", + "cudaMemcpyPeerAsync", + "cudaMemcpy3DPeer", + "cudaMemcpy3DPeerAsync", + "cudaDeviceReset", + "cudaDeviceSynchronize", + "cudaDeviceGetLimit", + "cudaDeviceSetLimit", + "cudaDeviceGetCacheConfig", + "cudaDeviceSetCacheConfig", + "cudaProfilerInitialize", + "cudaProfilerStart", + "cudaProfilerStop", + "cudaDeviceGetByPCIBusId", + "cudaDeviceGetPCIBusId", + "cudaGLGetDevices", + "cudaIpcGetEventHandle", + "cudaIpcOpenEventHandle", + "cudaIpcGetMemHandle", + "cudaIpcOpenMemHandle", + "cudaIpcCloseMemHandle", + "cudaArrayGetInfo", + "cudaFuncSetSharedMemConfig", + "cudaDeviceGetSharedMemConfig", + "cudaDeviceSetSharedMemConfig", + "cudaCreateTextureObject", + "cudaDestroyTextureObject", + "cudaGetTextureObjectResourceDesc", + "cudaGetTextureObjectTextureDesc", + "cudaCreateSurfaceObject", + "cudaDestroySurfaceObject", + "cudaGetSurfaceObjectResourceDesc", + "cudaMallocMipmappedArray", + "cudaGetMipmappedArrayLevel", + "cudaFreeMipmappedArray", + "cudaBindTextureToMipmappedArray", + "cudaGraphicsResourceGetMappedMipmappedArray", + "cudaStreamAddCallback", + "cudaStreamCreateWithFlags", + "cudaGetTextureObjectResourceViewDesc", + "cudaDeviceGetAttribute", + "cudaStreamDestroy", + "cudaStreamCreateWithPriority", + "cudaStreamGetPriority", + "cudaStreamGetFlags", + "cudaDeviceGetStreamPriorityRange", + "cudaMallocManaged", + "cudaOccupancyMaxActiveBlocksPerMultiprocessor", + "cudaStreamAttachMemAsync", + "cudaGetErrorName", + "cudaOccupancyMaxActiveBlocksPerMultiprocessor", + "cudaLaunchKernel", + "cudaGetDeviceFlags", + "cudaLaunch_ptsz", + "cudaLaunchKernel_ptsz", + "cudaMemcpy_ptds", + "cudaMemcpy2D_ptds", + "cudaMemcpyToArray_ptds", + "cudaMemcpy2DToArray_ptds", + "cudaMemcpyFromArray_ptds", + "cudaMemcpy2DFromArray_ptds", + "cudaMemcpyArrayToArray_ptds", + "cudaMemcpy2DArrayToArray_ptds", + "cudaMemcpyToSymbol_ptds", + "cudaMemcpyFromSymbol_ptds", + "cudaMemcpyAsync_ptsz", + "cudaMemcpyToArrayAsync_ptsz", + "cudaMemcpyFromArrayAsync_ptsz", + "cudaMemcpy2DAsync_ptsz", + "cudaMemcpy2DToArrayAsync_ptsz", + "cudaMemcpy2DFromArrayAsync_ptsz", + "cudaMemcpyToSymbolAsync_ptsz", + "cudaMemcpyFromSymbolAsync_ptsz", + "cudaMemset_ptds", + "cudaMemset2D_ptds", + "cudaMemsetAsync_ptsz", + "cudaMemset2DAsync_ptsz", + "cudaStreamGetPriority_ptsz", + "cudaStreamGetFlags_ptsz", + "cudaStreamSynchronize_ptsz", + "cudaStreamQuery_ptsz", + "cudaStreamAttachMemAsync_ptsz", + "cudaEventRecord_ptsz", + "cudaMemset3D_ptds", + "cudaMemset3DAsync_ptsz", + "cudaMemcpy3D_ptds", + "cudaMemcpy3DAsync_ptsz", + "cudaStreamWaitEvent_ptsz", + "cudaStreamAddCallback_ptsz", + "cudaMemcpy3DPeer_ptds", + "cudaMemcpy3DPeerAsync_ptsz", + "cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags", + "cudaMemPrefetchAsync", + "cudaMemPrefetchAsync_ptsz", + "cudaMemAdvise", + "cudaDeviceGetP2PAttribute", + "cudaGraphicsEGLRegisterImage", + "cudaEGLStreamConsumerConnect", + "cudaEGLStreamConsumerDisconnect", + "cudaEGLStreamConsumerAcquireFrame", + "cudaEGLStreamConsumerReleaseFrame", + "cudaEGLStreamProducerConnect", + "cudaEGLStreamProducerDisconnect", + "cudaEGLStreamProducerPresentFrame", + "cudaEGLStreamProducerReturnFrame", + "cudaGraphicsResourceGetMappedEglFrame", + "cudaMemRangeGetAttribute", + "cudaMemRangeGetAttributes", + "cudaEGLStreamConsumerConnectWithFlags", + "cudaLaunchCooperativeKernel", + "cudaLaunchCooperativeKernel_ptsz", + "cudaEventCreateFromEGLSync", + "cudaLaunchCooperativeKernelMultiDevice", + "cudaFuncSetAttribute", + "cudaImportExternalMemory", + "cudaExternalMemoryGetMappedBuffer", + "cudaExternalMemoryGetMappedMipmappedArray", + "cudaDestroyExternalMemory", + "cudaImportExternalSemaphore", + "cudaSignalExternalSemaphoresAsync", + "cudaSignalExternalSemaphoresAsync_ptsz", + "cudaWaitExternalSemaphoresAsync", + "cudaWaitExternalSemaphoresAsync_ptsz", + "cudaDestroyExternalSemaphore", + "cudaLaunchHostFunc", + "cudaLaunchHostFunc_ptsz", + "cudaGraphCreate", + "cudaGraphKernelNodeGetParams", + "cudaGraphKernelNodeSetParams", + "cudaGraphAddKernelNode", + "cudaGraphAddMemcpyNode", + "cudaGraphMemcpyNodeGetParams", + "cudaGraphMemcpyNodeSetParams", + "cudaGraphAddMemsetNode", + "cudaGraphMemsetNodeGetParams", + "cudaGraphMemsetNodeSetParams", + "cudaGraphAddHostNode", + "cudaGraphHostNodeGetParams", + "cudaGraphAddChildGraphNode", + "cudaGraphChildGraphNodeGetGraph", + "cudaGraphAddEmptyNode", + "cudaGraphClone", + "cudaGraphNodeFindInClone", + "cudaGraphNodeGetType", + "cudaGraphGetRootNodes", + "cudaGraphNodeGetDependencies", + "cudaGraphNodeGetDependentNodes", + "cudaGraphAddDependencies", + "cudaGraphRemoveDependencies", + "cudaGraphDestroyNode", + "cudaGraphInstantiate", + "cudaGraphLaunch", + "cudaGraphLaunch_ptsz", + "cudaGraphExecDestroy", + "cudaGraphDestroy", + "cudaStreamBeginCapture", + "cudaStreamBeginCapture_ptsz", + "cudaStreamIsCapturing", + "cudaStreamIsCapturing_ptsz", + "cudaStreamEndCapture", + "cudaStreamEndCapture_ptsz", + "cudaGraphHostNodeSetParams", + "cudaGraphGetNodes", + "cudaGraphGetEdges", + "cudaStreamGetCaptureInfo", + "cudaStreamGetCaptureInfo_ptsz", + "cudaGraphExecKernelNodeSetParams", + "cudaThreadExchangeStreamCaptureMode", + "cudaDeviceGetNvSciSyncAttributes", + "cudaOccupancyAvailableDynamicSMemPerBlock", + "cudaStreamSetFlags", + "cudaStreamSetFlags_ptsz", + "cudaGraphExecMemcpyNodeSetParams", + "cudaGraphExecMemsetNodeSetParams", + "cudaGraphExecHostNodeSetParams", + "cudaGraphExecUpdate", + "cudaGetFuncBySymbol", + "cudaCtxResetPersistingL2Cache", + "cudaGraphKernelNodeCopyAttributes", + "cudaGraphKernelNodeGetAttribute", + "cudaGraphKernelNodeSetAttribute", + "cudaStreamCopyAttributes", + "cudaStreamCopyAttributes_ptsz", + "cudaStreamGetAttribute", + "cudaStreamGetAttribute_ptsz", + "cudaStreamSetAttribute", + "cudaStreamSetAttribute_ptsz", + "cudaDeviceGetTexture1DLinearMaxWidth", + "cudaGraphUpload", + "cudaGraphUpload_ptsz", + "cudaGraphAddMemcpyNodeToSymbol", + "cudaGraphAddMemcpyNodeFromSymbol", + "cudaGraphAddMemcpyNode1D", + "cudaGraphMemcpyNodeSetParamsToSymbol", + "cudaGraphMemcpyNodeSetParamsFromSymbol", + "cudaGraphMemcpyNodeSetParams1D", + "cudaGraphExecMemcpyNodeSetParamsToSymbol", + "cudaGraphExecMemcpyNodeSetParamsFromSymbol", + "cudaGraphExecMemcpyNodeSetParams1D", + "cudaArrayGetSparseProperties", + "cudaMipmappedArrayGetSparseProperties", + "cudaGraphExecChildGraphNodeSetParams", + "cudaGraphAddEventRecordNode", + "cudaGraphEventRecordNodeGetEvent", + "cudaGraphEventRecordNodeSetEvent", + "cudaGraphAddEventWaitNode", + "cudaGraphEventWaitNodeGetEvent", + "cudaGraphEventWaitNodeSetEvent", + "cudaGraphExecEventRecordNodeSetEvent", + "cudaGraphExecEventWaitNodeSetEvent", + "cudaEventRecordWithFlags", + "cudaEventRecordWithFlags_ptsz", + "cudaDeviceGetDefaultMemPool", + "cudaMallocAsync", + "cudaMallocAsync_ptsz", + "cudaFreeAsync", + "cudaFreeAsync_ptsz", + "cudaMemPoolTrimTo", + "cudaMemPoolSetAttribute", + "cudaMemPoolGetAttribute", + "cudaMemPoolSetAccess", + "cudaArrayGetPlane", + "cudaMemPoolGetAccess", + "cudaMemPoolCreate", + "cudaMemPoolDestroy", + "cudaDeviceSetMemPool", + "cudaDeviceGetMemPool", + "cudaMemPoolExportToShareableHandle", + "cudaMemPoolImportFromShareableHandle", + "cudaMemPoolExportPointer", + "cudaMemPoolImportPointer", + "cudaMallocFromPoolAsync", + "cudaMallocFromPoolAsync_ptsz", + "cudaSignalExternalSemaphoresAsync", + "cudaSignalExternalSemaphoresAsync", + "cudaWaitExternalSemaphoresAsync", + "cudaWaitExternalSemaphoresAsync", + "cudaGraphAddExternalSemaphoresSignalNode", + "cudaGraphExternalSemaphoresSignalNodeGetParams", + "cudaGraphExternalSemaphoresSignalNodeSetParams", + "cudaGraphAddExternalSemaphoresWaitNode", + "cudaGraphExternalSemaphoresWaitNodeGetParams", + "cudaGraphExternalSemaphoresWaitNodeSetParams", + "cudaGraphExecExternalSemaphoresSignalNodeSetParams", + "cudaGraphExecExternalSemaphoresWaitNodeSetParams", + "SIZE" +}; + +const char* runtimeCbidName(CUpti_CallbackId cbid) { + constexpr int names_size = + sizeof(runtimeCbidNames) / sizeof(runtimeCbidNames[0]); + if (cbid < 0 || cbid >= names_size) { + return runtimeCbidNames[CUPTI_RUNTIME_TRACE_CBID_INVALID]; + } + return runtimeCbidNames[cbid]; +} + +} // namespace libkineto diff --git a/tb_plugins/profiling/libkineto/src/cupti_strings.h b/tb_plugins/profiling/libkineto/src/cupti_strings.h new file mode 100644 index 000000000..bbfebb983 --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/cupti_strings.h @@ -0,0 +1,14 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#pragma once + +#include + +namespace libkineto { + +const char* memoryKindString(CUpti_ActivityMemoryKind kind); +const char* memcpyKindString(CUpti_ActivityMemcpyKind kind); +const char* runtimeCbidName(CUpti_CallbackId cbid); +const char* overheadKindString(CUpti_ActivityOverheadKind kind); + +} // namespace libkineto diff --git a/tb_plugins/profiling/libkineto/src/init.cpp b/tb_plugins/profiling/libkineto/src/init.cpp new file mode 100644 index 000000000..4e1022485 --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/init.cpp @@ -0,0 +1,139 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#include +#include + +#include "ActivityProfilerProxy.h" +#include "Config.h" +#ifdef HAS_CUPTI +#include "CuptiCallbackApi.h" +#include "CuptiActivityApi.h" +#include "EventProfilerController.h" +#endif +#include "cupti_call.h" +#include "libkineto.h" + +#include "Logger.h" + +namespace KINETO_NAMESPACE { + +#ifdef HAS_CUPTI +static bool initialized = false; +static std::mutex initMutex; + +static void initProfilers( + CUpti_CallbackDomain /*domain*/, + CUpti_CallbackId /*cbid*/, + const CUpti_CallbackData* cbInfo) { + CUpti_ResourceData* d = (CUpti_ResourceData*)cbInfo; + CUcontext ctx = d->context; + + VLOG(0) << "CUDA Context created"; + std::lock_guard lock(initMutex); + + if (!initialized) { + libkineto::api().initProfilerIfRegistered(); + initialized = true; + VLOG(0) << "libkineto profilers activated"; + } + if (getenv("KINETO_DISABLE_EVENT_PROFILER") != nullptr) { + VLOG(0) << "Event profiler disabled via env var"; + } else { + ConfigLoader& config_loader = libkineto::api().configLoader(); + config_loader.initBaseConfig(); + EventProfilerController::start(ctx, config_loader); + } +} + +// Some models suffer from excessive instrumentation code gen +// on dynamic attach which can hang for more than 5+ seconds. +// If the workload was meant to be traced, preload the CUPTI +// to take the performance hit early on. +// https://docs.nvidia.com/cupti/r_main.html#r_overhead +static bool shouldPreloadCuptiInstrumentation() { + return getenv("PRELOAD_CUPTI_INSTRUMENTATION"); +} + +static void stopProfiler( + CUpti_CallbackDomain /*domain*/, + CUpti_CallbackId /*cbid*/, + const CUpti_CallbackData* cbInfo) { + CUpti_ResourceData* d = (CUpti_ResourceData*)cbInfo; + CUcontext ctx = d->context; + + LOG(INFO) << "CUDA Context destroyed"; + std::lock_guard lock(initMutex); + EventProfilerController::stop(ctx); +} +#endif // HAS_CUPTI + +} // namespace KINETO_NAMESPACE + +// Callback interface with CUPTI and library constructors +using namespace KINETO_NAMESPACE; +extern "C" { + +// Return true if no CUPTI errors occurred during init +bool libkineto_init(bool cpuOnly, bool logOnError) { + bool success = true; +#ifdef HAS_CUPTI + if (!cpuOnly) { + // libcupti will be lazily loaded on this call. + // If it is not available (e.g. CUDA is not installed), + // then this call will return an error and we just abort init. + auto& cbapi = CuptiCallbackApi::singleton(); + bool status = false; + + if (cbapi.initSuccess()){ + const CUpti_CallbackDomain domain = CUPTI_CB_DOMAIN_RESOURCE; + status = cbapi.registerCallback( + domain, CuptiCallbackApi::RESOURCE_CONTEXT_CREATED, initProfilers); + status = status && cbapi.registerCallback( + domain, CuptiCallbackApi::RESOURCE_CONTEXT_DESTROYED, stopProfiler); + + if (status) { + status = cbapi.enableCallback( + domain, CuptiCallbackApi::RESOURCE_CONTEXT_CREATED); + status = status && cbapi.enableCallback( + domain, CuptiCallbackApi::RESOURCE_CONTEXT_DESTROYED); + } + } + + if (!cbapi.initSuccess() || !status) { + success = false; + cpuOnly = true; + if (logOnError) { + CUPTI_CALL(cbapi.getCuptiStatus()); + LOG(WARNING) << "CUPTI initialization failed - " + << "CUDA profiler activities will be missing"; + LOG(INFO) << "If you see CUPTI_ERROR_INSUFFICIENT_PRIVILEGES, refer to " + << "https://developer.nvidia.com/nvidia-development-tools-solutions-err-nvgpuctrperm-cupti"; + } + } + } + + if (shouldPreloadCuptiInstrumentation()) { + CuptiActivityApi::forceLoadCupti(); + } +#endif // HAS_CUPTI + + ConfigLoader& config_loader = libkineto::api().configLoader(); + libkineto::api().registerProfiler( + std::make_unique(cpuOnly, config_loader)); + + return success; +} + +// The cuda driver calls this function if the CUDA_INJECTION64_PATH environment +// variable is set +int InitializeInjection(void) { + LOG(INFO) << "Injection mode: Initializing libkineto"; + libkineto_init(false /*cpuOnly*/, true /*logOnError*/); + return 1; +} + +void suppressLibkinetoLogMessages() { + SET_LOG_SEVERITY_LEVEL(ERROR); +} + +} // extern C diff --git a/tb_plugins/profiling/libkineto/src/libkineto_api.cpp b/tb_plugins/profiling/libkineto/src/libkineto_api.cpp new file mode 100644 index 000000000..9a622e4f5 --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/libkineto_api.cpp @@ -0,0 +1,41 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#include "libkineto.h" + +#include "ConfigLoader.h" +#include "ThreadUtil.h" + +namespace libkineto { + +LibkinetoApi& api() { + static LibkinetoApi instance(ConfigLoader::instance()); + return instance; +} + +void LibkinetoApi::initClientIfRegistered() { + if (client_) { + if (clientRegisterThread_ != threadId()) { + fprintf( + stderr, + "ERROR: External init callback must run in same thread as registerClient " + "(%d != %d)\n", + threadId(), + (int)clientRegisterThread_); + } else { + client_->init(); + } + } +} + +void LibkinetoApi::registerClient(ClientInterface* client) { + client_ = client; + if (client && activityProfiler_) { + // Can initialize straight away + client->init(); + } + // Assume here that the external init callback is *not* threadsafe + // and only call it if it's the same thread that called registerClient + clientRegisterThread_ = threadId(); +} + +} // namespace libkineto diff --git a/tb_plugins/profiling/libkineto/src/output_base.h b/tb_plugins/profiling/libkineto/src/output_base.h new file mode 100644 index 000000000..29d0d5776 --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/output_base.h @@ -0,0 +1,104 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#pragma once + +#include +#include +#include +#include +#include + +#ifdef HAS_CUPTI +#include +#include "CuptiActivity.h" +#endif // HAS_CUPTI +#include "ActivityBuffers.h" +#include "GenericTraceActivity.h" +#include "ThreadUtil.h" +#include "TraceSpan.h" + +namespace KINETO_NAMESPACE { + class Config; + class GpuKernelActivity; + struct RuntimeActivity; +} + +namespace libkineto { + +using namespace KINETO_NAMESPACE; + +class ActivityLogger { + public: + + virtual ~ActivityLogger() = default; + + struct DeviceInfo { + DeviceInfo(int64_t id, const std::string& name, const std::string& label) : + id(id), name(name), label(label) {} + int64_t id; + const std::string name; + const std::string label; + }; + + struct ResourceInfo { + ResourceInfo( + int64_t deviceId, + int64_t id, + int64_t sortIndex, + const std::string& name) : + id(id), sortIndex(sortIndex), deviceId(deviceId), name(name) {} + int64_t id; + int64_t sortIndex; + int64_t deviceId; + const std::string name; + }; + + struct OverheadInfo { + explicit OverheadInfo(const std::string& name) : name(name) {} + const std::string name; + }; + + virtual void handleDeviceInfo( + const DeviceInfo& info, + uint64_t time) = 0; + + virtual void handleResourceInfo(const ResourceInfo& info, int64_t time) = 0; + + virtual void handleOverheadInfo(const OverheadInfo& info, int64_t time) = 0; + + virtual void handleTraceSpan(const TraceSpan& span) = 0; + + virtual void handleActivity( + const libkineto::ITraceActivity& activity) = 0; + virtual void handleGenericActivity( + const libkineto::GenericTraceActivity& activity) = 0; + +#ifdef HAS_CUPTI + virtual void handleGpuActivity( + const GpuActivity& activity) = 0; + virtual void handleGpuActivity( + const GpuActivity& activity) = 0; + virtual void handleGpuActivity( + const GpuActivity& activity) = 0; + virtual void handleGpuActivity( + const GpuActivity& activity) = 0; +#endif // HAS_CUPTI + + virtual void handleTraceStart( + const std::unordered_map& metadata) = 0; + + void handleTraceStart() { + handleTraceStart(std::unordered_map()); + } + + virtual void finalizeTrace( + const KINETO_NAMESPACE::Config& config, + std::unique_ptr buffers, + int64_t endTime, + std::unordered_map>& metadata) = 0; + + protected: + ActivityLogger() = default; +}; + +} // namespace KINETO_NAMESPACE diff --git a/tb_plugins/profiling/libkineto/src/output_csv.cpp b/tb_plugins/profiling/libkineto/src/output_csv.cpp new file mode 100644 index 000000000..e56c02293 --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/output_csv.cpp @@ -0,0 +1,88 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#include "output_csv.h" + +#include +#include +#include + +#include +#include + +#include "Config.h" +#include "Logger.h" + +namespace KINETO_NAMESPACE { + +static void write_header( + std::ostream& out, + const std::vector& percentiles) { + out << "timestamp,delta_ms,device,event_name"; + for (int p : percentiles) { + out << ",p" << p; + } + out << ",total" << std::endl; +} + +void EventCSVLogger::update(const Config& config) { + eventNames_.clear(); + eventNames_.insert(config.eventNames().begin(), config.eventNames().end()); + eventNames_.insert(config.metricNames().begin(), config.metricNames().end()); + if (config.percentiles() != percentiles_) { + percentiles_ = config.percentiles(); + if (out_) { + write_header(*out_, percentiles_); + } + } +} + +void EventCSVLogger::handleSample(int device, const Sample& sample, bool from_new_version) { + using namespace std::chrono; + if (out_) { + auto now = system_clock::now(); + auto time = system_clock::to_time_t(now); + for (const Stat& s : sample.stats) { + if (eventNames_.find(s.name) == eventNames_.end()) { + continue; + } + *out_ << fmt::format("{:%Y-%m-%d %H:%M:%S}", fmt::localtime(time)) << ","; + *out_ << sample.deltaMsec << ","; + *out_ << device << ","; + *out_ << s.name; + for (const auto& p : s.percentileValues) { + *out_ << "," << p.second; + } + *out_ << "," << s.total << std::endl; + } + } +} + +void EventCSVFileLogger::update(const Config& config) { + if (config.eventLogFile() != filename_) { + if (of_.is_open()) { + of_.close(); + out_ = nullptr; + percentiles_.clear(); + } + filename_ = config.eventLogFile(); + if (!filename_.empty()) { + of_.open(filename_, std::ios::out | std::ios::trunc); + out_ = &of_; + } + } + EventCSVLogger::update(config); +} + +void EventCSVDbgLogger::update(const Config& config) { + if (out_ && config.verboseLogLevel() < 0) { + out_ = nullptr; + } else if (!out_ && config.verboseLogLevel() >= 0) { + out_ = &LIBKINETO_DBG_STREAM; + } + if (config.verboseLogLevel() >= 0) { + percentiles_.clear(); + EventCSVLogger::update(config); + } +} + +} // namespace KINETO_NAMESPACE diff --git a/tb_plugins/profiling/libkineto/src/output_csv.h b/tb_plugins/profiling/libkineto/src/output_csv.h new file mode 100644 index 000000000..bca29f4db --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/output_csv.h @@ -0,0 +1,39 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#pragma once +#include "SampleListener.h" + +#include +#include +#include + +namespace KINETO_NAMESPACE { + +class EventCSVLogger : public SampleListener { + public: + void update(const Config& config) override; + void handleSample(int device, const Sample& sample, bool from_new_version) override; + + protected: + EventCSVLogger() : out_(nullptr) {} + + std::ostream* out_; + std::set eventNames_; + std::vector percentiles_; +}; + +class EventCSVFileLogger : public EventCSVLogger { + public: + void update(const Config& config) override; + + private: + std::ofstream of_; + std::string filename_; +}; + +class EventCSVDbgLogger : public EventCSVLogger { + public: + void update(const Config& config) override; +}; + +} // namespace KINETO_NAMESPACE diff --git a/tb_plugins/profiling/libkineto/src/output_json.cpp b/tb_plugins/profiling/libkineto/src/output_json.cpp new file mode 100644 index 000000000..0ef22339f --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/output_json.cpp @@ -0,0 +1,583 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#include "output_json.h" + +#include +#include +#include +#include + +#include "Config.h" +#ifdef HAS_CUPTI +#include "CuptiActivity.h" +#include "CuptiActivity.tpp" +#include "CuptiActivityApi.h" +#include "CudaDeviceProperties.h" +#endif // HAS_CUPTI +#include "Demangle.h" +#include "TraceSpan.h" + +#include "Logger.h" + +using std::endl; +using namespace libkineto; + +namespace KINETO_NAMESPACE { + +static constexpr int kSchemaVersion = 1; +static constexpr char kFlowStart = 's'; +static constexpr char kFlowEnd = 'f'; + +#ifdef __linux__ +static constexpr char kDefaultLogFileFmt[] = + "/tmp/libkineto_activities_{}.json"; +#else +static constexpr char kDefaultLogFileFmt[] = "libkineto_activities_{}.json"; +#endif + +std::string& ChromeTraceLogger::sanitizeStrForJSON(std::string& value) { +// Replace all backslashes with forward slash because Windows paths causing JSONDecodeError. +#ifdef _WIN32 + std::replace(value.begin(), value.end(), '\\', '/'); +#endif + return value; +} + +void ChromeTraceLogger::metadataToJSON( + const std::unordered_map& metadata) { + for (const auto& kv : metadata) { + traceOf_ << fmt::format(R"JSON( + "{}": {},)JSON", kv.first, kv.second); + } +} + +void ChromeTraceLogger::handleTraceStart( + const std::unordered_map& metadata) { + traceOf_ << fmt::format(R"JSON( +{{ + "schemaVersion": {},)JSON", kSchemaVersion); + +#ifdef HAS_CUPTI + traceOf_ << fmt::format(R"JSON( + "deviceProperties": [{} + ],)JSON", devicePropertiesJson()); +#endif + + metadataToJSON(metadata); + traceOf_ << R"JSON( + "traceEvents": [)JSON"; +} + +static std::string defaultFileName() { + return fmt::format(kDefaultLogFileFmt, processId()); +} + +void ChromeTraceLogger::openTraceFile() { + traceOf_.open(fileName_, std::ofstream::out | std::ofstream::trunc); + if (!traceOf_) { + PLOG(ERROR) << "Failed to open '" << fileName_ << "'"; + } else { + LOG(INFO) << "Tracing to " << fileName_; + } +} + +ChromeTraceLogger::ChromeTraceLogger(const std::string& traceFileName) { + fileName_ = traceFileName.empty() ? defaultFileName() : traceFileName; + traceOf_.clear(std::ios_base::badbit); + openTraceFile(); +} + +static int64_t us(int64_t timestamp) { + // It's important that this conversion is the same here and in the CPU trace. + // No rounding! + return timestamp / 1000; +} + +void ChromeTraceLogger::handleDeviceInfo( + const DeviceInfo& info, + uint64_t time) { + if (!traceOf_) { + return; + } + + // M is for metadata + // process_name needs a pid and a name arg + // clang-format off + traceOf_ << fmt::format(R"JSON( + {{ + "name": "process_name", "ph": "M", "ts": {}, "pid": {}, "tid": 0, + "args": {{ + "name": "{}" + }} + }}, + {{ + "name": "process_labels", "ph": "M", "ts": {}, "pid": {}, "tid": 0, + "args": {{ + "labels": "{}" + }} + }}, + {{ + "name": "process_sort_index", "ph": "M", "ts": {}, "pid": {}, "tid": 0, + "args": {{ + "sort_index": {} + }} + }},)JSON", + time, info.id, + info.name, + time, info.id, + info.label, + time, info.id, + info.id < 8 ? info.id + 0x1000000ll : info.id); + // clang-format on +} + +void ChromeTraceLogger::handleResourceInfo( + const ResourceInfo& info, + int64_t time) { + if (!traceOf_) { + return; + } + + // M is for metadata + // thread_name needs a pid and a name arg + // clang-format off + traceOf_ << fmt::format(R"JSON( + {{ + "name": "thread_name", "ph": "M", "ts": {}, "pid": {}, "tid": {}, + "args": {{ + "name": "{}" + }} + }}, + {{ + "name": "thread_sort_index", "ph": "M", "ts": {}, "pid": {}, "tid": {}, + "args": {{ + "sort_index": {} + }} + }},)JSON", + time, info.deviceId, info.id, + info.name, + time, info.deviceId, info.id, + info.sortIndex); + // clang-format on +} + +void ChromeTraceLogger::handleOverheadInfo( + const OverheadInfo& info, + int64_t time) { + if (!traceOf_) { + return; + } + + // TOOD: reserve pid = -1 for overhead but we need to rethink how to scale this for + // other metadata + // clang-format off + traceOf_ << fmt::format(R"JSON( + {{ + "name": "process_name", "ph": "M", "ts": {}, "pid": -1, "tid": 0, + "args": {{ + "name": "{}" + }} + }}, + {{ + "name": "process_sort_index", "ph": "M", "ts": {}, "pid": -1, "tid": 0, + "args": {{ + "sort_index": {} + }} + }},)JSON", + time, + info.name, + time, + 0x100000All); + // clang-format on +} + +void ChromeTraceLogger::handleTraceSpan(const TraceSpan& span) { + if (!traceOf_) { + return; + } + + // clang-format off + traceOf_ << fmt::format(R"JSON( + {{ + "ph": "X", "cat": "Trace", "ts": {}, "dur": {}, + "pid": "Spans", "tid": "{}", + "name": "{}{} ({})", + "args": {{ + "Op count": {} + }} + }}, + {{ + "name": "process_sort_index", "ph": "M", "ts": {}, + "pid": "Spans", "tid": 0, + "args": {{ + "sort_index": {} + }} + }},)JSON", + span.startTime, span.endTime - span.startTime, + span.name, + span.prefix, span.name, span.iteration, + span.opCount, + span.startTime, + // Large sort index to appear at the bottom + 0x20000000ll); + // clang-format on + + addIterationMarker(span); +} + +void ChromeTraceLogger::addIterationMarker(const TraceSpan& span) { + if (!traceOf_) { + return; + } + + // clang-format off + traceOf_ << fmt::format(R"JSON( + {{ + "name": "Iteration Start: {}", "ph": "i", "s": "g", + "pid": "Traces", "tid": "Trace {}", "ts": {} + }},)JSON", + span.name, + span.name, span.startTime); + // clang-format on +} + +static std::string traceActivityJson(const ITraceActivity& activity) { + // clang-format off + int64_t ts = activity.timestamp(); + int64_t duration = activity.duration(); + if (activity.type() == ActivityType::GPU_USER_ANNOTATION) { + // The GPU user annotations start at the same time as the + // first associated GPU activity. Since they appear later + // in the trace file, this causes a visualization issue in Chrome. + // Make it start one us earlier. + ts--; + duration++; // Still need it to end at the orginal point + } + return fmt::format(R"JSON( + "name": "{}", "pid": {}, "tid": {}, + "ts": {}, "dur": {})JSON", + activity.name(), activity.deviceId(), activity.resourceId(), + ts, duration); + // clang-format on +} + +void ChromeTraceLogger::handleGenericInstantEvent( + const libkineto::ITraceActivity& op) { + if (!traceOf_) { + return; + } + + traceOf_ << fmt::format(R"JSON( + {{ + "ph": "i", "s": "t", "name": "{}", + "pid": {}, "tid": {}, + "ts": {}, + "args": {{ + {} + }} + }},)JSON", + op.name(), op.deviceId(), op.resourceId(), + op.timestamp(), op.metadataJson()); +} + +void ChromeTraceLogger::handleActivity( + const libkineto::ITraceActivity& op) { + if (!traceOf_) { + return; + } + + if (op.type() == ActivityType::CPU_INSTANT_EVENT) { + handleGenericInstantEvent(op); + return; + } + + const std::string op_metadata = op.metadataJson(); + std::string separator = ""; + if (op_metadata.find_first_not_of(" \t\n") != std::string::npos) { + separator = ",\n "; + } + std::string span = ""; + if (op.traceSpan()) { + span = fmt::format(R"JSON( + "Trace name": "{}", "Trace iteration": {},)JSON", + op.traceSpan()->name, + op.traceSpan()->iteration); + } + + // clang-format off + traceOf_ << fmt::format(R"JSON( + {{ + "ph": "X", "cat": "{}", {}, + "args": {{{} + "External id": {}{}{} + }} + }},)JSON", + toString(op.type()), traceActivityJson(op), + // args + span, + op.correlationId(), separator, op_metadata); + // clang-format on + if (op.flowId() > 0) { + handleGenericLink(op); + } +} + +void ChromeTraceLogger::handleGenericActivity( + const libkineto::GenericTraceActivity& op) { + handleActivity(op); +} + +void ChromeTraceLogger::handleGenericLink(const ITraceActivity& act) { + static struct { + int type; + char longName[24]; + char shortName[16]; + } flow_names[] = { + {kLinkFwdBwd, "forward_backward", "fwd_bwd"}, + {kLinkAsyncCpuGpu, "async_cpu_to_gpu", "async_gpu"} + }; + for (auto& flow : flow_names) { + if (act.flowType() == flow.type) { + // Link the activities via flow ID in source and destination. + // The source node must return true from flowStart() + // and the destination node false. + if (act.flowStart()) { + handleLink(kFlowStart, act, act.flowId(), flow.longName, flow.shortName); + } else { + handleLink(kFlowEnd, act, act.flowId(), flow.longName, flow.shortName); + } + return; + } + } + LOG(ERROR) << "Unknown flow type: " << act.flowType(); +} + +void ChromeTraceLogger::handleLink( + char type, + const ITraceActivity& e, + int64_t id, + const std::string& cat, + const std::string& name) { + if (!traceOf_) { + return; + } + + // clang-format off + traceOf_ << fmt::format(R"JSON( + {{ + "ph": "{}", "id": {}, "pid": {}, "tid": {}, "ts": {}, + "cat": "{}", "name": "{}", "bp": "e" + }},)JSON", + type, id, e.deviceId(), e.resourceId(), e.timestamp(), cat, name); + // clang-format on +} + +#ifdef HAS_CUPTI +// GPU side kernel activity +void ChromeTraceLogger::handleGpuActivity( + const GpuActivity& activity) { + if (!traceOf_) { + return; + } + const CUpti_ActivityKernel4* kernel = &activity.raw(); + constexpr int threads_per_warp = 32; + float blocks_per_sm = -1.0; + float warps_per_sm = -1.0; + int sm_count = smCount(kernel->deviceId); + if (sm_count) { + blocks_per_sm = + (kernel->gridX * kernel->gridY * kernel->gridZ) / (float) sm_count; + warps_per_sm = + blocks_per_sm * (kernel->blockX * kernel->blockY * kernel->blockZ) + / threads_per_warp; + } + + // Calculate occupancy + float occupancy = KINETO_NAMESPACE::kernelOccupancy( + kernel->deviceId, + kernel->registersPerThread, + kernel->staticSharedMemory, + kernel->dynamicSharedMemory, + kernel->blockX, + kernel->blockY, + kernel->blockZ, + blocks_per_sm); + + // clang-format off + traceOf_ << fmt::format(R"JSON( + {{ + "ph": "X", "cat": "Kernel", {}, + "args": {{ + "queued": {}, "device": {}, "context": {}, + "stream": {}, "correlation": {}, + "registers per thread": {}, + "shared memory": {}, + "blocks per SM": {}, + "warps per SM": {}, + "grid": [{}, {}, {}], + "block": [{}, {}, {}], + "est. achieved occupancy %": {} + }} + }},)JSON", + traceActivityJson(activity), + // args + us(kernel->queued), kernel->deviceId, kernel->contextId, + kernel->streamId, kernel->correlationId, + kernel->registersPerThread, + kernel->staticSharedMemory + kernel->dynamicSharedMemory, + blocks_per_sm, + warps_per_sm, + kernel->gridX, kernel->gridY, kernel->gridZ, + kernel->blockX, kernel->blockY, kernel->blockZ, + (int) (0.5 + occupancy * 100.0)); + // clang-format on + + auto to_id = activity.correlationId(); + handleLink(kFlowEnd, activity, to_id, "async_cpu_to_gpu", "async_gpu"); +} + +static std::string bandwidth(uint64_t bytes, uint64_t duration) { + return duration == 0 ? "\"N/A\"" : fmt::format("{}", bytes * 1.0 / duration); +} + +// GPU side memcpy activity +void ChromeTraceLogger::handleGpuActivity( + const GpuActivity& activity) { + if (!traceOf_) { + return; + } + const CUpti_ActivityMemcpy& memcpy = activity.raw(); + VLOG(2) << memcpy.correlationId << ": MEMCPY"; + // clang-format off + traceOf_ << fmt::format(R"JSON( + {{ + "ph": "X", "cat": "Memcpy", {}, + "args": {{ + "device": {}, "context": {}, + "stream": {}, "correlation": {}, + "bytes": {}, "memory bandwidth (GB/s)": {} + }} + }},)JSON", + traceActivityJson(activity), + // args + memcpy.deviceId, memcpy.contextId, + memcpy.streamId, memcpy.correlationId, + memcpy.bytes, bandwidth(memcpy.bytes, memcpy.end - memcpy.start)); + // clang-format on + + int64_t to_id = activity.correlationId(); + handleLink(kFlowEnd, activity, to_id, "async_cpu_to_gpu", "async_gpu"); +} + +// GPU side memcpy activity +void ChromeTraceLogger::handleGpuActivity( + const GpuActivity& activity) { + if (!traceOf_) { + return; + } + const CUpti_ActivityMemcpy2& memcpy = activity.raw(); + // clang-format off + traceOf_ << fmt::format(R"JSON( + {{ + "ph": "X", "cat": "Memcpy", {}, + "args": {{ + "fromDevice": {}, "inDevice": {}, "toDevice": {}, + "fromContext": {}, "inContext": {}, "toContext": {}, + "stream": {}, "correlation": {}, + "bytes": {}, "memory bandwidth (GB/s)": {} + }} + }},)JSON", + traceActivityJson(activity), + // args + memcpy.srcDeviceId, memcpy.deviceId, memcpy.dstDeviceId, + memcpy.srcContextId, memcpy.contextId, memcpy.dstContextId, + memcpy.streamId, memcpy.correlationId, + memcpy.bytes, bandwidth(memcpy.bytes, memcpy.end - memcpy.start)); + // clang-format on + + int64_t to_id = activity.correlationId(); + handleLink(kFlowEnd, activity, to_id, "async_cpu_to_gpu", "async_gpu"); +} + +void ChromeTraceLogger::handleGpuActivity( + const GpuActivity& activity) { + if (!traceOf_) { + return; + } + const CUpti_ActivityMemset& memset = activity.raw(); + // clang-format off + traceOf_ << fmt::format(R"JSON( + {{ + "ph": "X", "cat": "Memset", {}, + "args": {{ + "device": {}, "context": {}, + "stream": {}, "correlation": {}, + "bytes": {}, "memory bandwidth (GB/s)": {} + }} + }},)JSON", + traceActivityJson(activity), + // args + memset.deviceId, memset.contextId, + memset.streamId, memset.correlationId, + memset.bytes, bandwidth(memset.bytes, memset.end - memset.start)); + // clang-format on + + int64_t to_id = activity.correlationId(); + handleLink(kFlowEnd, activity, to_id, "async_cpu_to_gpu", "async_gpu"); +} +#endif // HAS_CUPTI + +void ChromeTraceLogger::finalizeTrace( + const Config& /*unused*/, + std::unique_ptr /*unused*/, + int64_t endTime, + std::unordered_map>& metadata) { + if (!traceOf_) { + LOG(ERROR) << "Failed to write to log file!"; + return; + } + LOG(INFO) << "Chrome Trace written to " << fileName_; + // clang-format off + traceOf_ << fmt::format(R"JSON( + {{ + "name": "Record Window End", "ph": "i", "s": "g", + "pid": "", "tid": "", "ts": {} + }} + ],)JSON", + endTime); + +#if !USE_GOOGLE_LOG + std::unordered_map PreparedMetadata; + for (const auto& kv : metadata) { + // Skip empty log buckets, ex. skip ERROR if its empty. + if (!kv.second.empty()) { + std::string value = "["; + // Ex. Each metadata from logger is a list of strings, expressed in JSON as + // "ERROR": ["Error 1", "Error 2"], + // "WARNING": ["Warning 1", "Warning 2", "Warning 3"], + // ... + int mdv_count = kv.second.size(); + for (const auto& v : kv.second) { + value.append("\"" + v + "\""); + if(mdv_count > 1) { + value.append(","); + mdv_count--; + } + } + value.append("]"); + PreparedMetadata[kv.first] = sanitizeStrForJSON(value); + } + } + metadataToJSON(PreparedMetadata); +#endif // !USE_GOOGLE_LOG + + // Putting this here because the last entry MUST not end with a comma. + traceOf_ << fmt::format(R"JSON( + "traceName": "{}" +}})JSON", sanitizeStrForJSON(fileName_)); + // clang-format on + + traceOf_.close(); +} + +} // namespace KINETO_NAMESPACE diff --git a/tb_plugins/profiling/libkineto/src/output_json.h b/tb_plugins/profiling/libkineto/src/output_json.h new file mode 100644 index 000000000..5a8a81e4a --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/output_json.h @@ -0,0 +1,91 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#pragma once + +#include +#include +#include +#include +#include + +#ifdef HAS_CUPTI +#include +#endif +#include "GenericTraceActivity.h" +#include "output_base.h" + +namespace KINETO_NAMESPACE { + // Previous declaration of TraceSpan is struct. Must match the same here. + struct TraceSpan; +} + +namespace KINETO_NAMESPACE { + +class Config; + +class ChromeTraceLogger : public libkineto::ActivityLogger { + public: + explicit ChromeTraceLogger(const std::string& traceFileName); + + // Note: the caller of these functions should handle concurrency + // i.e., we these functions are not thread-safe + void handleDeviceInfo( + const DeviceInfo& info, + uint64_t time) override; + + void handleOverheadInfo(const OverheadInfo& info, int64_t time) override; + + void handleResourceInfo(const ResourceInfo& info, int64_t time) override; + + void handleTraceSpan(const TraceSpan& span) override; + + void handleActivity(const ITraceActivity& activity) override; + void handleGenericActivity(const GenericTraceActivity& activity) override; + +#ifdef HAS_CUPTI + void handleGpuActivity(const GpuActivity& activity) override; + void handleGpuActivity(const GpuActivity& activity) override; + void handleGpuActivity(const GpuActivity& activity) override; + void handleGpuActivity(const GpuActivity& activity) override; +#endif // HAS_CUPTI + + void handleTraceStart( + const std::unordered_map& metadata) override; + + void finalizeTrace( + const Config& config, + std::unique_ptr buffers, + int64_t endTime, + std::unordered_map>& metadata) override; + + std::string traceFileName() const { + return fileName_; + } + + private: + + // Create a flow event (arrow) + void handleLink( + char type, + const ITraceActivity& e, + int64_t id, + const std::string& cat, + const std::string& name); + + void addIterationMarker(const TraceSpan& span); + + void openTraceFile(); + + void handleGenericInstantEvent(const ITraceActivity& op); + + void handleGenericLink(const ITraceActivity& activity); + + void metadataToJSON(const std::unordered_map& metadata); + + std::string& sanitizeStrForJSON(std::string& value); + + std::string fileName_; + std::ofstream traceOf_; +}; + +} // namespace KINETO_NAMESPACE diff --git a/tb_plugins/profiling/libkineto/src/output_membuf.h b/tb_plugins/profiling/libkineto/src/output_membuf.h new file mode 100644 index 000000000..ef6aadeb6 --- /dev/null +++ b/tb_plugins/profiling/libkineto/src/output_membuf.h @@ -0,0 +1,130 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#pragma once + +#include +#include +#include +#include + +#ifdef HAS_CUPTI +#include +#endif + +#include "Config.h" +#include "GenericTraceActivity.h" +#ifdef HAS_CUPTI +#include "CuptiActivity.h" +#include "CuptiActivity.tpp" +#endif // HAS_CUPTI +#include "output_base.h" + +namespace KINETO_NAMESPACE { + +class Config; + +class MemoryTraceLogger : public ActivityLogger { + public: + MemoryTraceLogger(const Config& config) : config_(config.clone()) { + activities_.reserve(100000); + } + + // Note: the caller of these functions should handle concurrency + // i.e., these functions are not thread-safe + void handleDeviceInfo( + const DeviceInfo& info, + uint64_t time) override { + deviceInfoList_.emplace_back(info, time); + } + + void handleResourceInfo(const ResourceInfo& info, int64_t time) override { + resourceInfoList_.emplace_back(info, time); + } + + void handleOverheadInfo(const OverheadInfo& info, int64_t time) override {} + + void handleTraceSpan(const TraceSpan& span) override { + // Handled separately + } + + template + void addActivityWrapper(const T& act) { + wrappers_.push_back(std::make_unique(act)); + activities_.push_back(wrappers_.back().get()); + } + + // Just add the pointer to the list - ownership of the underlying + // objects must be transferred in ActivityBuffers via finalizeTrace + void handleActivity(const ITraceActivity& activity) override { + activities_.push_back(&activity); + } + void handleGenericActivity(const GenericTraceActivity& activity) override { + addActivityWrapper(activity); + } + +#ifdef HAS_CUPTI + void handleGpuActivity(const GpuActivity& activity) override { + addActivityWrapper(activity); + } + void handleGpuActivity(const GpuActivity& activity) override { + addActivityWrapper(activity); + } + void handleGpuActivity(const GpuActivity& activity) override { + addActivityWrapper(activity); + } + void handleGpuActivity(const GpuActivity& activity) override { + addActivityWrapper(activity); + } +#endif // HAS_CUPTI + + void handleTraceStart( + const std::unordered_map& metadata) override { + metadata_ = metadata; + } + + void finalizeTrace( + const Config& config, + std::unique_ptr buffers, + int64_t endTime, + std::unordered_map>& metadata) override { + buffers_ = std::move(buffers); + endTime_ = endTime; + } + + const std::vector* traceActivities() { + return &activities_; + } + + void log(ActivityLogger& logger) { + logger.handleTraceStart(metadata_); + for (auto& activity : activities_) { + activity->log(logger); + } + for (auto& p : deviceInfoList_) { + logger.handleDeviceInfo(p.first, p.second); + } + for (auto& p : resourceInfoList_) { + logger.handleResourceInfo(p.first, p.second); + } + for (auto& cpu_trace_buffer : buffers_->cpu) { + logger.handleTraceSpan(cpu_trace_buffer->span); + } + // Hold on to the buffers + logger.finalizeTrace(*config_, nullptr, endTime_, loggerMetadata_); + } + + private: + + std::unique_ptr config_; + // Optimization: Remove unique_ptr by keeping separate vector per type + std::vector activities_; + std::vector> wrappers_; + std::vector> deviceInfoList_; + std::vector> resourceInfoList_; + std::unique_ptr buffers_; + std::unordered_map metadata_; + std::unordered_map> loggerMetadata_; + int64_t endTime_{0}; +}; + +} // namespace KINETO_NAMESPACE diff --git a/tb_plugins/profiling/libkineto/test/CMakeLists.txt b/tb_plugins/profiling/libkineto/test/CMakeLists.txt new file mode 100644 index 000000000..ca54460b3 --- /dev/null +++ b/tb_plugins/profiling/libkineto/test/CMakeLists.txt @@ -0,0 +1,3 @@ +cmake_minimum_required(VERSION 3.5 FATAL_ERROR) + +# TODO diff --git a/tb_plugins/profiling/libkineto/test/ConfigTest.cpp b/tb_plugins/profiling/libkineto/test/ConfigTest.cpp new file mode 100644 index 000000000..16bc86e75 --- /dev/null +++ b/tb_plugins/profiling/libkineto/test/ConfigTest.cpp @@ -0,0 +1,315 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#include "include/Config.h" + +#include +#include +#include +#include + +using namespace std::chrono; +using namespace KINETO_NAMESPACE; + +TEST(ParseTest, Whitespace) { + Config cfg; + // Check that various types of whitespace is ignored + EXPECT_TRUE(cfg.parse("")); + EXPECT_TRUE(cfg.parse(" ")); + EXPECT_TRUE(cfg.parse("\t")); + EXPECT_TRUE(cfg.parse("\n")); + EXPECT_TRUE(cfg.parse(" ")); + EXPECT_TRUE(cfg.parse("\t \n \t\t\n\n")); + // Only the above characters are supported + EXPECT_FALSE(cfg.parse("\r\n")); +} + +TEST(ParseTest, Comment) { + Config cfg; + // Anything following a '#' should be ignored, up to a newline + EXPECT_TRUE(cfg.parse("# comment")); + EXPECT_TRUE(cfg.parse(" # ~!@#$")); + EXPECT_TRUE(cfg.parse("\t#abc")); + EXPECT_TRUE(cfg.parse("###\n##")); + EXPECT_TRUE(cfg.parse("EVENTS=util ##ok")); + EXPECT_TRUE(cfg.parse("EVENTS=util ## EVENTS=instruction")); + // Whatever appears before the comment must be valid format + EXPECT_FALSE(cfg.parse("util ## not ok")); + EXPECT_FALSE(cfg.parse("## ok \n blah # not OK")); + // Check that a comment does not affect config parsing + EXPECT_TRUE(cfg.parse("SAMPLE_PERIOD_MSECS = 1 # Sample every millisecond")); + EXPECT_EQ(cfg.samplePeriod(), milliseconds(1)); +} + +TEST(ParseTest, Format) { + Config cfg; + // The basic format is just "name = value". + // Where both value and name can be almost anything. + // Leading and trailing whitespace should be removed + // for both 'name' and 'value', but internal whitespace is not. + EXPECT_FALSE(cfg.parse("events")); + EXPECT_TRUE(cfg.parse("events=")); + EXPECT_FALSE(cfg.parse("=events=")); + EXPECT_TRUE(cfg.parse("events=1,2,3")); + // Only one setting per line + EXPECT_FALSE(cfg.parse("events = 1,2,3 ; metrics = 4,5,6")); + // Names are case sensitive + EXPECT_TRUE(cfg.parse("EVENTS = 1,2,3 \n metrics = 4,5,6")); + EXPECT_EQ(cfg.eventNames(), std::set({"1", "2", "3"})); + EXPECT_EQ(cfg.metricNames().size(), 0); + // Leading and trailing whitespace removed for event and metric names, + // but not internal. + EXPECT_TRUE( + cfg.parse("EVENTS = 1, 2, 3 \n \tMETRICS\t = \t4,\t5\t,\ts i x ")); + EXPECT_EQ(cfg.eventNames(), std::set({"1", "2", "3"})); + EXPECT_EQ(cfg.metricNames(), std::set({"4", "5", "s i x"})); +} + +TEST(ParseTest, DefaultActivityTypes) { + Config cfg; + cfg.validate(std::chrono::system_clock::now()); + auto all_activities = activityTypes(); + // TODO: introduce optional activities + EXPECT_EQ(cfg.selectedActivityTypes(), + std::set(all_activities.begin(), all_activities.end() - 1)); +} + +TEST(ParseTest, ActivityTypes) { + Config cfg; + EXPECT_FALSE(cfg.parse("ACTIVITY_TYPES")); + EXPECT_TRUE(cfg.parse("ACTIVITY_TYPES=")); + EXPECT_FALSE(cfg.parse("=ACTIVITY_TYPES=")); + + EXPECT_EQ(cfg.selectedActivityTypes(), + std::set({ActivityType::CPU_OP, + ActivityType::CPU_INSTANT_EVENT, + ActivityType::PYTHON_FUNCTION, + ActivityType::USER_ANNOTATION, + ActivityType::GPU_USER_ANNOTATION, + ActivityType::GPU_MEMCPY, + ActivityType::GPU_MEMSET, + ActivityType::CONCURRENT_KERNEL, + ActivityType::EXTERNAL_CORRELATION, + ActivityType::GLOW_RUNTIME, + ActivityType::CUDA_RUNTIME, + ActivityType::CUDA_PROFILER_RANGE})); + + Config cfg2; + EXPECT_TRUE(cfg2.parse("ACTIVITY_TYPES=gpu_memcpy,gpu_MeMsEt,kernel")); + EXPECT_EQ(cfg2.selectedActivityTypes(), + std::set({ActivityType::GPU_MEMCPY, + ActivityType::GPU_MEMSET, + ActivityType::CONCURRENT_KERNEL})); + + EXPECT_TRUE(cfg2.parse("ACTIVITY_TYPES = cuda_Runtime,")); + EXPECT_EQ(cfg2.selectedActivityTypes(), + std::set({ActivityType::CUDA_RUNTIME})); + + // Should throw an exception because incorrect activity name + EXPECT_FALSE(cfg2.parse("ACTIVITY_TYPES = memcopy,cuda_runtime")); + + EXPECT_TRUE(cfg2.parse("ACTIVITY_TYPES = cpu_op")); + EXPECT_EQ(cfg2.selectedActivityTypes(), + std::set({ActivityType::CPU_OP})); +} + +TEST(ParseTest, SamplePeriod) { + Config cfg; + EXPECT_TRUE(cfg.parse("SAMPLE_PERIOD_MSECS=10")); + EXPECT_EQ(cfg.samplePeriod(), milliseconds(10)); + EXPECT_TRUE(cfg.parse("SAMPLE_PERIOD_MSECS=0")); + cfg.validate(std::chrono::system_clock::now()); + // 0 should be adjustd up to 1 + EXPECT_EQ(cfg.samplePeriod(), milliseconds(1)); + // Negative and non-int values should fail + EXPECT_FALSE(cfg.parse("SAMPLE_PERIOD_MSECS=-10")); + EXPECT_FALSE(cfg.parse("SAMPLE_PERIOD_MSECS=1.5")); + EXPECT_FALSE(cfg.parse("SAMPLE_PERIOD_MSECS=")); + EXPECT_FALSE(cfg.parse("SAMPLE_PERIOD_MSECS=string")); + EXPECT_EQ(cfg.samplePeriod(), milliseconds(1)); +} + +TEST(ParseTest, MultiplexPeriod) { + Config cfg; + auto now = std::chrono::system_clock::now(); + + EXPECT_TRUE(cfg.parse("SAMPLE_PERIOD_MSECS=100\nMULTIPLEX_PERIOD_MSECS=100")); + EXPECT_EQ(cfg.multiplexPeriod(), milliseconds(100)); + EXPECT_TRUE(cfg.parse("MULTIPLEX_PERIOD_MSECS = 0")); + cfg.validate(now); + // Adjusted to match sample period + EXPECT_EQ(cfg.multiplexPeriod(), milliseconds(100)); + EXPECT_TRUE(cfg.parse("MULTIPLEX_PERIOD_MSECS \t= \t 750 \n")); + cfg.validate(now); + // Adjusted to match multiple of sample period + EXPECT_EQ(cfg.multiplexPeriod(), milliseconds(800)); + EXPECT_FALSE(cfg.parse("MULTIPLEX_PERIOD_MSECS=-10")); + EXPECT_FALSE(cfg.parse("MULTIPLEX_PERIOD_MSECS=1.5")); + EXPECT_FALSE(cfg.parse("MULTIPLEX_PERIOD_MSECS=")); + EXPECT_FALSE(cfg.parse("MULTIPLEX_PERIOD_MSECS=string")); + // Previous value not affected + EXPECT_EQ(cfg.multiplexPeriod(), milliseconds(800)); +} + +TEST(ParseTest, ReportPeriod) { + Config cfg; + EXPECT_TRUE(cfg.parse("REPORT_PERIOD_SECS=1")); + EXPECT_EQ(cfg.reportPeriod(), seconds(1)); + // Whitespace + EXPECT_TRUE(cfg.parse("REPORT_PERIOD_SECS = \t100")); + EXPECT_EQ(cfg.reportPeriod(), seconds(100)); + // Invalid types + EXPECT_FALSE(cfg.parse("REPORT_PERIOD_SECS=-1")); + EXPECT_EQ(cfg.reportPeriod(), seconds(100)); +} + +TEST(ParseTest, SamplesPerReport) { + Config cfg; + auto now = std::chrono::system_clock::now(); + + EXPECT_TRUE(cfg.parse(R"( + SAMPLE_PERIOD_MSECS = 1000 + REPORT_PERIOD_SECS = 1 + SAMPLES_PER_REPORT = 10)")); + cfg.validate(now); + // Adjusted down to one sample per report + EXPECT_EQ(cfg.samplesPerReport(), 1); + EXPECT_TRUE(cfg.parse(R"( + SAMPLE_PERIOD_MSECS = 1000 + REPORT_PERIOD_SECS = 10 + SAMPLES_PER_REPORT = 10)")); + cfg.validate(now); + // No adjustment needed + EXPECT_EQ(cfg.samplesPerReport(), 10); + EXPECT_TRUE(cfg.parse(R"( + SAMPLE_PERIOD_MSECS = 1000 + REPORT_PERIOD_SECS = 2 + SAMPLES_PER_REPORT = 10)")); + cfg.validate(now); + // Adjusted to 2 samples per report + EXPECT_EQ(cfg.samplesPerReport(), 2); + EXPECT_TRUE(cfg.parse(R"( + SAMPLE_PERIOD_MSECS = 200 + REPORT_PERIOD_SECS = 2 + SAMPLES_PER_REPORT = 10)")); + cfg.validate(now); + // No adjustment needed + EXPECT_EQ(cfg.samplesPerReport(), 10); + EXPECT_TRUE(cfg.parse("SAMPLES_PER_REPORT=0")); + cfg.validate(now); + // Adjusted up to 1 + EXPECT_EQ(cfg.samplesPerReport(), 1); + // Invalid value types + EXPECT_FALSE(cfg.parse("SAMPLES_PER_REPORT=-10")); + EXPECT_FALSE(cfg.parse("SAMPLES_PER_REPORT=1.5")); + EXPECT_EQ(cfg.samplesPerReport(), 1); + + EXPECT_TRUE(cfg.parse(R"( + SAMPLE_PERIOD_MSECS=1000 + MULTIPLEX_PERIOD_MSECS=500 # Must be a multiple of sample period + REPORT_PERIOD_SECS=0 # Must be non-zero multiple of multiplex period + SAMPLES_PER_REPORT=5 # Max report period / multiplex period)")); + cfg.validate(now); + // Multiple adjustments + EXPECT_EQ(cfg.samplePeriod(), milliseconds(1000)); + EXPECT_EQ(cfg.multiplexPeriod(), milliseconds(1000)); + EXPECT_EQ(cfg.reportPeriod(), seconds(1)); + EXPECT_EQ(cfg.samplesPerReport(), 1); +} + +TEST(ParseTest, EnableSigUsr2) { + Config cfg; + EXPECT_TRUE(cfg.parse("ENABLE_SIGUSR2=yes")); + EXPECT_TRUE(cfg.sigUsr2Enabled()); + EXPECT_TRUE(cfg.parse("ENABLE_SIGUSR2=no")); + EXPECT_FALSE(cfg.sigUsr2Enabled()); + EXPECT_TRUE(cfg.parse("ENABLE_SIGUSR2=YES")); + EXPECT_TRUE(cfg.sigUsr2Enabled()); + EXPECT_TRUE(cfg.parse("ENABLE_SIGUSR2=NO")); + EXPECT_FALSE(cfg.sigUsr2Enabled()); + EXPECT_TRUE(cfg.parse("ENABLE_SIGUSR2=Y")); + EXPECT_TRUE(cfg.sigUsr2Enabled()); + EXPECT_TRUE(cfg.parse("ENABLE_SIGUSR2=N")); + EXPECT_FALSE(cfg.sigUsr2Enabled()); + EXPECT_TRUE(cfg.parse("ENABLE_SIGUSR2=T")); + EXPECT_TRUE(cfg.sigUsr2Enabled()); + EXPECT_TRUE(cfg.parse("ENABLE_SIGUSR2=F")); + EXPECT_FALSE(cfg.sigUsr2Enabled()); + EXPECT_TRUE(cfg.parse("ENABLE_SIGUSR2=true")); + EXPECT_TRUE(cfg.sigUsr2Enabled()); + EXPECT_TRUE(cfg.parse("ENABLE_SIGUSR2=false")); + EXPECT_FALSE(cfg.sigUsr2Enabled()); + EXPECT_FALSE(cfg.parse("ENABLE_SIGUSR2= ")); + EXPECT_FALSE(cfg.parse("ENABLE_SIGUSR2=2")); + EXPECT_FALSE(cfg.parse("ENABLE_SIGUSR2=-1")); + EXPECT_FALSE(cfg.parse("ENABLE_SIGUSR2=yep")); +} + +TEST(ParseTest, DeviceMask) { + Config cfg; + // Single device + EXPECT_TRUE(cfg.parse("EVENTS_ENABLED_DEVICES = 0")); + EXPECT_TRUE(cfg.eventProfilerEnabledForDevice(0)); + EXPECT_FALSE(cfg.eventProfilerEnabledForDevice(1)); + + // Two devices, internal whitespace + EXPECT_TRUE(cfg.parse("EVENTS_ENABLED_DEVICES = 1, 2")); + EXPECT_FALSE(cfg.eventProfilerEnabledForDevice(0)); + EXPECT_TRUE(cfg.eventProfilerEnabledForDevice(1)); + EXPECT_TRUE(cfg.eventProfilerEnabledForDevice(2)); + EXPECT_FALSE(cfg.eventProfilerEnabledForDevice(3)); + + // Three devices, check that previous devices are ignored + EXPECT_TRUE(cfg.parse("EVENTS_ENABLED_DEVICES = 0, 2,4")); + EXPECT_TRUE(cfg.eventProfilerEnabledForDevice(0)); + EXPECT_FALSE(cfg.eventProfilerEnabledForDevice(1)); + EXPECT_TRUE(cfg.eventProfilerEnabledForDevice(2)); + EXPECT_FALSE(cfg.eventProfilerEnabledForDevice(3)); + EXPECT_TRUE(cfg.eventProfilerEnabledForDevice(4)); + EXPECT_FALSE(cfg.eventProfilerEnabledForDevice(5)); + + // Repeated numbers have no effect + EXPECT_TRUE(cfg.parse("EVENTS_ENABLED_DEVICES = 0,1,1,1,2,3,2,1,3,7,7,3")); + EXPECT_TRUE(cfg.eventProfilerEnabledForDevice(0)); + EXPECT_TRUE(cfg.eventProfilerEnabledForDevice(1)); + EXPECT_TRUE(cfg.eventProfilerEnabledForDevice(2)); + EXPECT_TRUE(cfg.eventProfilerEnabledForDevice(3)); + EXPECT_FALSE(cfg.eventProfilerEnabledForDevice(4)); + EXPECT_FALSE(cfg.eventProfilerEnabledForDevice(6)); + EXPECT_TRUE(cfg.eventProfilerEnabledForDevice(7)); + + // 8 is larger than the max allowed + EXPECT_FALSE(cfg.parse("EVENTS_ENABLED_DEVICES = 3,8")); + + // 300 cannot be held in an uint8_t + EXPECT_FALSE(cfg.parse("EVENTS_ENABLED_DEVICES = 300")); + + // Various illegal cases + EXPECT_FALSE(cfg.parse("EVENTS_ENABLED_DEVICES = 0,1,two,three")); + EXPECT_FALSE(cfg.parse("EVENTS_ENABLED_DEVICES = 0,1,,2")); + EXPECT_FALSE(cfg.parse("EVENTS_ENABLED_DEVICES = -1")); + EXPECT_FALSE(cfg.parse("EVENTS_ENABLED_DEVICES = 1.0")); +} + +TEST(ParseTest, RequestTime) { + Config cfg; + system_clock::time_point now = system_clock::now(); + int64_t tgood_ms = + duration_cast(now.time_since_epoch()).count(); + EXPECT_TRUE(cfg.parse(fmt::format("REQUEST_TIMESTAMP = {}", tgood_ms))); + + tgood_ms = duration_cast((now - seconds(5)).time_since_epoch()) + .count(); + EXPECT_TRUE(cfg.parse(fmt::format("REQUEST_TIMESTAMP = {}", tgood_ms))); + + int64_t tbad_ms = + duration_cast((now - seconds(20)).time_since_epoch()) + .count(); + EXPECT_FALSE(cfg.parse(fmt::format("REQUEST_TIMESTAMP = {}", tbad_ms))); + + EXPECT_FALSE(cfg.parse("REQUEST_TIMESTAMP = 0")); + EXPECT_FALSE(cfg.parse("REQUEST_TIMESTAMP = -1")); + + tbad_ms = duration_cast((now + seconds(10)).time_since_epoch()) + .count(); + EXPECT_FALSE(cfg.parse(fmt::format("REQUEST_TIMESTAMP = {}", tbad_ms))); +} diff --git a/tb_plugins/profiling/libkineto/test/CuptiActivityProfilerTest.cpp b/tb_plugins/profiling/libkineto/test/CuptiActivityProfilerTest.cpp new file mode 100644 index 000000000..6e67980ee --- /dev/null +++ b/tb_plugins/profiling/libkineto/test/CuptiActivityProfilerTest.cpp @@ -0,0 +1,629 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#include +#include +#include +#include +#include +#include + +#ifdef __linux__ +#include +#include +#include +#endif + +#include "include/libkineto.h" +#include "include/Config.h" +#include "src/CuptiActivityProfiler.h" +#include "src/ActivityTrace.h" +#include "src/CuptiActivityApi.h" +#include "src/output_base.h" +#include "src/output_json.h" +#include "src/output_membuf.h" + +#include "src/Logger.h" +#include "test/MockActivitySubProfiler.h" + +using namespace std::chrono; +using namespace KINETO_NAMESPACE; + +#define CUDA_LAUNCH_KERNEL CUPTI_RUNTIME_TRACE_CBID_cudaLaunchKernel_v7000 +#define CUDA_MEMCPY CUPTI_RUNTIME_TRACE_CBID_cudaMemcpy_v3020 + +namespace { +const TraceSpan& defaultTraceSpan() { + static TraceSpan span(0, 0, "Unknown", ""); + return span; +} +} + +// Provides ability to easily create a few test CPU-side ops +struct MockCpuActivityBuffer : public CpuTraceBuffer { + MockCpuActivityBuffer(int64_t startTime, int64_t endTime) { + span = TraceSpan(startTime, endTime,"Test trace"); + gpuOpCount = 0; + } + + void addOp(std::string name, int64_t startTime, int64_t endTime, int64_t correlation) { + GenericTraceActivity op(span, ActivityType::CPU_OP, name); + op.startTime = startTime; + op.endTime = endTime; + op.resource = systemThreadId(); + op.id = correlation; + activities.push_back(std::move(op)); + span.opCount++; + } +}; + +// Provides ability to easily create a few test CUPTI ops +struct MockCuptiActivityBuffer { + void addCorrelationActivity(int64_t correlation, CUpti_ExternalCorrelationKind externalKind, int64_t externalId) { + auto& act = *(CUpti_ActivityExternalCorrelation*) malloc(sizeof(CUpti_ActivityExternalCorrelation)); + act.kind = CUPTI_ACTIVITY_KIND_EXTERNAL_CORRELATION; + act.externalId = externalId; + act.externalKind = externalKind; + act.correlationId = correlation; + activities.push_back(reinterpret_cast(&act)); + } + + void addRuntimeActivity( + CUpti_runtime_api_trace_cbid_enum cbid, + int64_t start_us, int64_t end_us, int64_t correlation) { + auto& act = createActivity( + start_us, end_us, correlation); + act.kind = CUPTI_ACTIVITY_KIND_RUNTIME; + act.cbid = cbid; + act.threadId = threadId(); + activities.push_back(reinterpret_cast(&act)); + } + + void addKernelActivity( + int64_t start_us, int64_t end_us, int64_t correlation) { + auto& act = createActivity( + start_us, end_us, correlation); + act.kind = CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL; + act.deviceId = 0; + act.streamId = 1; + act.name = "kernel"; + act.gridX = act.gridY = act.gridZ = 1; + act.blockX = act.blockY = act.blockZ = 1; + activities.push_back(reinterpret_cast(&act)); + } + + void addMemcpyActivity( + int64_t start_us, int64_t end_us, int64_t correlation) { + auto& act = createActivity( + start_us, end_us, correlation); + act.kind = CUPTI_ACTIVITY_KIND_MEMCPY; + act.deviceId = 0; + act.streamId = 2; + act.copyKind = CUPTI_ACTIVITY_MEMCPY_KIND_HTOD; + act.srcKind = CUPTI_ACTIVITY_MEMORY_KIND_PINNED; + act.dstKind = CUPTI_ACTIVITY_MEMORY_KIND_DEVICE; + activities.push_back(reinterpret_cast(&act)); + } + + template + T& createActivity( + int64_t start_us, int64_t end_us, int64_t correlation) { + T& act = *static_cast(malloc(sizeof(T))); + bzero(&act, sizeof(act)); + act.start = start_us * 1000; + act.end = end_us * 1000; + act.correlationId = correlation; + return act; + } + + ~MockCuptiActivityBuffer() { + for (CUpti_Activity* act : activities) { + free(act); + } + } + + std::vector activities; +}; + +// Mock parts of the CuptiActivityApi +class MockCuptiActivities : public CuptiActivityApi { + public: + virtual int smCount() override { + return 10; + } + + virtual const std::pair processActivities( + CuptiActivityBufferMap&, /*unused*/ + std::function handler) override { + for (CUpti_Activity* act : activityBuffer->activities) { + handler(act); + } + return {activityBuffer->activities.size(), 100}; + } + + virtual std::unique_ptr + activityBuffers() override { + auto map = std::make_unique(); + auto buf = std::make_unique(100); + uint8_t* addr = buf->data(); + (*map)[addr] = std::move(buf); + return map; + } + + void bufferRequestedOverride(uint8_t** buffer, size_t* size, size_t* maxNumRecords) { + this->bufferRequested(buffer, size, maxNumRecords); + } + + std::unique_ptr activityBuffer; +}; + + +// Common setup / teardown and helper functions +class CuptiActivityProfilerTest : public ::testing::Test { + protected: + void SetUp() override { + profiler_ = std::make_unique( + cuptiActivities_, /*cpu only*/ false); + cfg_ = std::make_unique(); + cfg_->validate(std::chrono::system_clock::now()); + loggerFactory.addProtocol("file", [](const std::string& url) { + return std::unique_ptr(new ChromeTraceLogger(url)); + }); + } + + std::unique_ptr cfg_; + MockCuptiActivities cuptiActivities_; + std::unique_ptr profiler_; + ActivityLoggerFactory loggerFactory; +}; + +void checkTracefile(const char* filename) { +#ifdef __linux__ + // Check that the expected file was written and that it has some content + int fd = open(filename, O_RDONLY); + if (!fd) { + perror(filename); + } + EXPECT_TRUE(fd); + // Should expect at least 100 bytes + struct stat buf{}; + fstat(fd, &buf); + EXPECT_GT(buf.st_size, 100); + close(fd); +#endif +} + +TEST(CuptiActivityProfiler, AsyncTrace) { + std::vector log_modules( + {"CuptiActivityProfiler.cpp", "output_json.cpp"}); + SET_LOG_VERBOSITY_LEVEL(1, log_modules); + + MockCuptiActivities activities; + CuptiActivityProfiler profiler(activities, /*cpu only*/ true); + + char filename[] = "/tmp/libkineto_testXXXXXX.json"; + mkstemps(filename, 5); + + Config cfg; + + int iter = 0; + int warmup = 5; + auto now = system_clock::now(); + auto startTime = now + seconds(10); + + bool success = cfg.parse(fmt::format(R"CFG( + ACTIVITIES_WARMUP_PERIOD_SECS = {} + ACTIVITIES_DURATION_SECS = 1 + ACTIVITIES_LOG_FILE = {} + PROFILE_START_TIME = {} + )CFG", warmup, filename, duration_cast(startTime.time_since_epoch()).count())); + + EXPECT_TRUE(success); + EXPECT_FALSE(profiler.isActive()); + + auto logger = std::make_unique(cfg.activitiesLogFile()); + + // Usually configuration is done when now is startTime - warmup to kick off warmup + // but start right away in the test + profiler.configure(cfg, now); + profiler.setLogger(logger.get()); + + EXPECT_TRUE(profiler.isActive()); + + // fast forward in time and we have reached the startTime + now = startTime; + + // Run the profiler + // Warmup + // performRunLoopStep is usually called by the controller loop and takes + // the current time and the controller's next wakeup time. + profiler.performRunLoopStep( + /* Current time */ now, /* Next wakeup time */ now); + + auto next = now + milliseconds(1000); + + // performRunLoopStep can also be called by an application thread to update iteration count + // since this config does not use iteration this should have no effect on the state + while (++iter < 20) { + profiler.performRunLoopStep(now, now, iter); + } + + // Runloop should now be in collect state, so start workload + // Perform another runloop step, passing in the end profile time as current. + // This should terminate collection + profiler.performRunLoopStep( + /* Current time */ next, /* Next wakeup time */ next); + // One step needed for each of the Process and Finalize phases + // Doesn't really matter what times we pass in here. + + EXPECT_TRUE(profiler.isActive()); + + auto nextnext = next + milliseconds(1000); + + while (++iter < 40) { + profiler.performRunLoopStep(next, next, iter); + } + + EXPECT_TRUE(profiler.isActive()); + + profiler.performRunLoopStep(nextnext,nextnext); + profiler.performRunLoopStep(nextnext,nextnext); + + // Assert that tracing has completed + EXPECT_FALSE(profiler.isActive()); + + checkTracefile(filename); +} + +TEST(CuptiActivityProfiler, AsyncTraceUsingIter) { + std::vector log_modules( + {"CuptiActivityProfiler.cpp", "output_json.cpp"}); + SET_LOG_VERBOSITY_LEVEL(1, log_modules); + + auto runIterTest = [&]( + int start_iter, int warmup_iters, int trace_iters) { + + LOG(INFO ) << "Async Trace Test: start_iteration = " << start_iter + << " warmup iterations = " << warmup_iters + << " trace iterations = " << trace_iters; + + MockCuptiActivities activities; + CuptiActivityProfiler profiler(activities, /*cpu only*/ true); + + char filename[] = "/tmp/libkineto_testXXXXXX.json"; + mkstemps(filename, 5); + + Config cfg; + + int iter = 0; + auto now = system_clock::now(); + + bool success = cfg.parse(fmt::format(R"CFG( + PROFILE_START_ITERATION = {} + ACTIVITIES_WARMUP_ITERATIONS={} + ACTIVITIES_ITERATIONS={} + ACTIVITIES_DURATION_SECS = 1 + ACTIVITIES_LOG_FILE = {} + )CFG", start_iter, warmup_iters, trace_iters, filename)); + + EXPECT_TRUE(success); + EXPECT_FALSE(profiler.isActive()); + + auto logger = std::make_unique(cfg.activitiesLogFile()); + + // Usually configuration is done when now is startIter - warmup iter to kick off warmup + // but start right away in the test + while (iter < (start_iter - warmup_iters)) { + profiler.performRunLoopStep(now, now, iter++); + } + + profiler.configure(cfg, now); + profiler.setLogger(logger.get()); + + EXPECT_TRUE(profiler.isActive()); + + // fast forward in time, mimicking what will happen in reality + now += seconds(10); + auto next = now + milliseconds(1000); + + // this call to runloop step should not be effecting the state + profiler.performRunLoopStep(now, next); + EXPECT_TRUE(profiler.isActive()); + + // start trace collection + while (iter < start_iter) { + profiler.performRunLoopStep(now, next, iter++); + } + + // Runloop should now be in collect state, so start workload + + while (iter < (start_iter + trace_iters)) { + profiler.performRunLoopStep(now, next, iter++); + } + + // One step is required for each of the Process and Finalize phases + // Doesn't really matter what times we pass in here. + if (iter >= (start_iter + trace_iters)) { + profiler.performRunLoopStep(now, next, iter++); + } + EXPECT_TRUE(profiler.isActive()); + + auto nextnext = next + milliseconds(1000); + + profiler.performRunLoopStep(nextnext, nextnext); + profiler.performRunLoopStep(nextnext, nextnext); + + // Assert that tracing has completed + EXPECT_FALSE(profiler.isActive()); + + checkTracefile(filename); + }; + + // start iter = 50, warmup iters = 5, trace iters = 10 + runIterTest(50, 5, 10); + // should be able to start at 0 iteration + runIterTest(0, 0, 2); + runIterTest(0, 5, 5); +} + +TEST_F(CuptiActivityProfilerTest, SyncTrace) { + using ::testing::Return; + using ::testing::ByMove; + + // Verbose logging is useful for debugging + std::vector log_modules( + {"CuptiActivityProfiler.cpp"}); + SET_LOG_VERBOSITY_LEVEL(2, log_modules); + + // Start and stop profiling + CuptiActivityProfiler profiler(cuptiActivities_, /*cpu only*/ false); + int64_t start_time_us = 100; + int64_t duration_us = 300; + auto start_time = time_point(microseconds(start_time_us)); + profiler.configure(*cfg_, start_time); + profiler.startTrace(start_time); + profiler.stopTrace(start_time + microseconds(duration_us)); + + profiler.recordThreadInfo(); + + // Log some cpu ops + auto cpuOps = std::make_unique( + start_time_us, start_time_us + duration_us); + cpuOps->addOp("op1", 120, 150, 1); + cpuOps->addOp("op2", 130, 140, 2); + cpuOps->addOp("op3", 200, 250, 3); + profiler.transferCpuTrace(std::move(cpuOps)); + + // And some GPU ops + auto gpuOps = std::make_unique(); + gpuOps->addRuntimeActivity(CUDA_LAUNCH_KERNEL, 133, 138, 1); + gpuOps->addRuntimeActivity(CUDA_MEMCPY, 210, 220, 2); + gpuOps->addRuntimeActivity(CUDA_LAUNCH_KERNEL, 230, 245, 3); + gpuOps->addKernelActivity(150, 170, 1); + gpuOps->addMemcpyActivity(240, 250, 2); + gpuOps->addKernelActivity(260, 320, 3); + cuptiActivities_.activityBuffer = std::move(gpuOps); + + // Have the profiler process them + auto logger = std::make_unique(*cfg_); + profiler.processTrace(*logger); + + // Profiler can be reset at this point - logger owns the activities + profiler_->reset(); + + // Wrapper that allows iterating over the activities + ActivityTrace trace(std::move(logger), loggerFactory); + EXPECT_EQ(trace.activities()->size(), 9); + std::map activityCounts; + std::map resourceIds; + for (auto& activity : *trace.activities()) { + activityCounts[activity->name()]++; + resourceIds[activity->resourceId()]++; + } + for (const auto& p : activityCounts) { + LOG(INFO) << p.first << ": " << p.second; + } + EXPECT_EQ(activityCounts["op1"], 1); + EXPECT_EQ(activityCounts["op2"], 1); + EXPECT_EQ(activityCounts["op3"], 1); + EXPECT_EQ(activityCounts["cudaLaunchKernel"], 2); + EXPECT_EQ(activityCounts["cudaMemcpy"], 1); + EXPECT_EQ(activityCounts["kernel"], 2); + EXPECT_EQ(activityCounts["Memcpy HtoD (Pinned -> Device)"], 1); + + auto sysTid = systemThreadId(); + // Ops and runtime events are on thread sysTid + EXPECT_EQ(resourceIds[sysTid], 6); + // Kernels are on stream 1, memcpy on stream 2 + EXPECT_EQ(resourceIds[1], 2); + EXPECT_EQ(resourceIds[2], 1); + +#ifdef __linux__ + char filename[] = "/tmp/libkineto_testXXXXXX.json"; + mkstemps(filename, 5); + trace.save(filename); + // Check that the expected file was written and that it has some content + int fd = open(filename, O_RDONLY); + if (!fd) { + perror(filename); + } + EXPECT_TRUE(fd); + // Should expect at least 100 bytes + struct stat buf{}; + fstat(fd, &buf); + EXPECT_GT(buf.st_size, 100); +#endif +} + +TEST_F(CuptiActivityProfilerTest, GpuUserAnnotationTest) { + // Verbose logging is useful for debugging + std::vector log_modules( + {"CuptiActivityProfiler.cpp"}); + SET_LOG_VERBOSITY_LEVEL(2, log_modules); + + // Start and stop profiling + CuptiActivityProfiler profiler(cuptiActivities_, /*cpu only*/ false); + int64_t start_time_us = 100; + int64_t duration_us = 300; + auto start_time = time_point(microseconds(start_time_us)); + profiler.configure(*cfg_, start_time); + profiler.startTrace(start_time); + profiler.stopTrace(start_time + microseconds(duration_us)); + + int64_t kernelLaunchTime = 120; + profiler.recordThreadInfo(); + + // set up CPU event + auto cpuOps = std::make_unique( + start_time_us, start_time_us + duration_us); + cpuOps->addOp("annotation", kernelLaunchTime, kernelLaunchTime + 10, 1); + profiler.transferCpuTrace(std::move(cpuOps)); + + // set up a couple of GPU events and correlate with above CPU event. + // CUPTI_EXTERNAL_CORRELATION_KIND_CUSTOM1 is used for user annotations. + auto gpuOps = std::make_unique(); + gpuOps->addCorrelationActivity(1, CUPTI_EXTERNAL_CORRELATION_KIND_CUSTOM1, 1); + gpuOps->addKernelActivity(kernelLaunchTime + 5, kernelLaunchTime + 10, 1); + gpuOps->addCorrelationActivity(1, CUPTI_EXTERNAL_CORRELATION_KIND_CUSTOM1, 1); + gpuOps->addKernelActivity(kernelLaunchTime + 15, kernelLaunchTime + 25, 1); + cuptiActivities_.activityBuffer = std::move(gpuOps); + + // process trace + auto logger = std::make_unique(*cfg_); + profiler.processTrace(*logger); + + ActivityTrace trace(std::move(logger), loggerFactory); + std::map counts; + for (auto& activity : *trace.activities()) { + counts[activity->name()]++; + } + + // We should now have an additional annotation activity created + // on the GPU timeline. + EXPECT_EQ(counts["annotation"], 2); + EXPECT_EQ(counts["kernel"], 2); + + auto& annotation = trace.activities()->at(0); + auto& kernel1 = trace.activities()->at(1); + auto& kernel2 = trace.activities()->at(2); + auto& gpu_annotation = trace.activities()->at(3); + EXPECT_EQ(gpu_annotation->type(), ActivityType::GPU_USER_ANNOTATION); + EXPECT_EQ(gpu_annotation->timestamp(), kernel1->timestamp()); + EXPECT_EQ( + gpu_annotation->duration(), + kernel2->timestamp() + kernel2->duration() - kernel1->timestamp()); + EXPECT_EQ(gpu_annotation->deviceId(), kernel1->deviceId()); + EXPECT_EQ(gpu_annotation->resourceId(), kernel1->resourceId()); + EXPECT_EQ(gpu_annotation->correlationId(), annotation->correlationId()); + EXPECT_EQ(gpu_annotation->name(), annotation->name()); +} + +TEST_F(CuptiActivityProfilerTest, SubActivityProfilers) { + using ::testing::Return; + using ::testing::ByMove; + + // Verbose logging is useful for debugging + std::vector log_modules( + {"CuptiActivityProfiler.cpp"}); + SET_LOG_VERBOSITY_LEVEL(2, log_modules); + + // Setup example events to test + GenericTraceActivity ev{defaultTraceSpan(), ActivityType::GLOW_RUNTIME, ""}; + ev.device = 1; + ev.resource = 0; + + int64_t start_time_us = 100; + int64_t duration_us = 1000; + auto start_time = time_point(microseconds(start_time_us)); + + std::vector test_activities{3, ev}; + test_activities[0].startTime = start_time_us; + test_activities[0].endTime = start_time_us + 5000; + test_activities[0].activityName = "SubGraph A execution"; + test_activities[1].startTime = start_time_us; + test_activities[1].endTime = start_time_us + 2000; + test_activities[1].activityName = "Operator foo"; + test_activities[2].startTime = start_time_us + 2500; + test_activities[2].endTime = start_time_us + 2900; + test_activities[2].activityName = "Operator bar"; + + auto mock_activity_profiler = + std::make_unique(test_activities); + + MockCuptiActivities activities; + CuptiActivityProfiler profiler(activities, /*cpu only*/ true); + profiler.addChildActivityProfiler( + std::move(mock_activity_profiler)); + + profiler.configure(*cfg_, start_time); + profiler.startTrace(start_time); + EXPECT_TRUE(profiler.isActive()); + + profiler.stopTrace(start_time + microseconds(duration_us)); + EXPECT_TRUE(profiler.isActive()); + + char filename[] = "/tmp/libkineto_testXXXXXX.json"; + mkstemps(filename, 5); + LOG(INFO) << "Logging to tmp file " << filename; + + // process trace + auto logger = std::make_unique(*cfg_); + profiler.processTrace(*logger); + profiler.setLogger(logger.get()); + + ActivityTrace trace(std::move(logger), loggerFactory); + trace.save(filename); + const auto& traced_activites = trace.activities(); + + // Test we have all the events + EXPECT_EQ(traced_activites->size(), test_activities.size()); + + // Check that the expected file was written and that it has some content + int fd = open(filename, O_RDONLY); + if (!fd) { + perror(filename); + } + EXPECT_TRUE(fd); + + // Should expect at least 100 bytes + struct stat buf{}; + fstat(fd, &buf); + EXPECT_GT(buf.st_size, 100); +} + +TEST_F(CuptiActivityProfilerTest, BufferSizeLimitTestWarmup) { + CuptiActivityProfiler profiler(cuptiActivities_, /*cpu only*/ false); + + auto now = system_clock::now(); + auto startTime = now + seconds(10); + + int maxBufferSizeMB = 3; + + auto startTimeEpoch = std::to_string(duration_cast(startTime.time_since_epoch()).count()); + std::string maxBufferSizeMBStr = std::to_string(maxBufferSizeMB); + cfg_->handleOption("ACTIVITIES_MAX_GPU_BUFFER_SIZE_MB", maxBufferSizeMBStr); + cfg_->handleOption("PROFILE_START_TIME", startTimeEpoch); + + + EXPECT_FALSE(profiler.isActive()); + profiler.configure(*cfg_, now); + EXPECT_TRUE(profiler.isActive()); + + for (size_t i = 0; i < maxBufferSizeMB; i++) { + uint8_t* buf; + size_t gpuBufferSize; + size_t maxNumRecords; + cuptiActivities_.bufferRequestedOverride(&buf, &gpuBufferSize, &maxNumRecords); + } + + // fast forward to startTime and profiler is now running + now = startTime; + + profiler.performRunLoopStep(now, now); + + auto next = now + milliseconds(1000); + profiler.performRunLoopStep(next, next); + profiler.performRunLoopStep(next, next); + profiler.performRunLoopStep(next, next); + + EXPECT_FALSE(profiler.isActive()); +} diff --git a/tb_plugins/profiling/libkineto/test/CuptiCallbackApiTest.cpp b/tb_plugins/profiling/libkineto/test/CuptiCallbackApiTest.cpp new file mode 100644 index 000000000..253b696da --- /dev/null +++ b/tb_plugins/profiling/libkineto/test/CuptiCallbackApiTest.cpp @@ -0,0 +1,239 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#include "src/Logger.h" +#include "src/CuptiCallbackApi.h" + +#include +#include +#include +#include + +using namespace std::chrono; +using namespace KINETO_NAMESPACE; +using namespace libkineto; + +const size_t some_data = 42; + +std::atomic simple_cb_calls = 0; + +void simple_cb( + CUpti_CallbackDomain domain, + CUpti_CallbackId cbid, + const CUpti_CallbackData* cbInfo) { + + // simple arg check + EXPECT_EQ(domain, CUPTI_CB_DOMAIN_RUNTIME_API); + EXPECT_EQ(cbid, CUPTI_RUNTIME_TRACE_CBID_cudaLaunchKernel_v7000); + EXPECT_EQ(*reinterpret_cast(cbInfo), some_data); + + simple_cb_calls++; +} + +void atomic_cb( + CUpti_CallbackDomain /*domain*/, + CUpti_CallbackId /*cbid*/, + const CUpti_CallbackData* /*cbInfo)*/) { + // do some atomics in a loop + for (int i = 0; i < 1000; i++) { + // would have used release consistency but this is fine + simple_cb_calls++; + } +} + +void empty_cb( + CUpti_CallbackDomain /*domain*/, + CUpti_CallbackId /*cbid*/, + const CUpti_CallbackData* /*cbInfo*/) { +} + +TEST(CuptiCallbackApiTest, SimpleTest) { + auto& api = CuptiCallbackApi::singleton(); + + auto addSimpleCallback = [&]() -> bool { + bool ret = api.registerCallback( + CUPTI_CB_DOMAIN_RUNTIME_API, + CuptiCallbackApi::CUDA_LAUNCH_KERNEL, + &simple_cb + ); + return ret; + }; + EXPECT_TRUE(addSimpleCallback()) << "Failed to add callback"; + + // duplicate add should be okay + EXPECT_TRUE(addSimpleCallback()) << "Failed to re-add callback"; + + simple_cb_calls = 0; + + // simulate callback + api.__callback_switchboard( + CUPTI_CB_DOMAIN_RUNTIME_API, + CUPTI_RUNTIME_TRACE_CBID_cudaLaunchKernel_v7000, + reinterpret_cast(&some_data)); + + EXPECT_EQ(simple_cb_calls, 1); + + bool ret = api.deleteCallback( + CUPTI_CB_DOMAIN_RUNTIME_API, + CuptiCallbackApi::CUDA_LAUNCH_KERNEL, + &simple_cb + ); + + EXPECT_TRUE(ret) << "Failed to remove callback"; + + ret = api.deleteCallback( + CUPTI_CB_DOMAIN_RUNTIME_API, + CuptiCallbackApi::CUDA_LAUNCH_KERNEL, + &atomic_cb + ); + + EXPECT_FALSE(ret) << "oops! deleted a callback that was never added"; +} + +TEST(CuptiCallbackApiTest, AllCallbacks) { + auto& api = CuptiCallbackApi::singleton(); + + auto testCallback = [&]( + CUpti_CallbackDomain domain, + CUpti_CallbackId cbid, + CuptiCallbackApi::CuptiCallBackID kineto_cbid) -> bool { + + bool ret = api.registerCallback(domain, kineto_cbid, atomic_cb); + EXPECT_TRUE(ret) << "Failed to add callback"; + + if (!ret) { + return false; + } + + simple_cb_calls = 0; + api.__callback_switchboard(domain, cbid, nullptr); + EXPECT_EQ(simple_cb_calls, 1000); + ret = simple_cb_calls == 1000; + + EXPECT_TRUE(api.deleteCallback(domain, kineto_cbid, atomic_cb)); + + return ret; + }; + + EXPECT_TRUE( + testCallback( + CUPTI_CB_DOMAIN_RESOURCE, + CUPTI_CBID_RESOURCE_CONTEXT_CREATED, + CuptiCallbackApi::RESOURCE_CONTEXT_CREATED)) + << "Failed to run callback for RESOURCE_CONTEXT_CREATED"; + + EXPECT_TRUE( + testCallback( + CUPTI_CB_DOMAIN_RESOURCE, + CUPTI_CBID_RESOURCE_CONTEXT_DESTROY_STARTING, + CuptiCallbackApi::RESOURCE_CONTEXT_DESTROYED)) + << "Failed to run callback for RESOURCE_CONTEXT_DESTROYED"; + + EXPECT_TRUE( + testCallback( + CUPTI_CB_DOMAIN_RUNTIME_API, + CUPTI_RUNTIME_TRACE_CBID_cudaLaunchKernel_v7000, + CuptiCallbackApi::CUDA_LAUNCH_KERNEL)) + << "Failed to run callback for CUDA_LAUNCH_KERNEL"; + +} + +TEST(CuptiCallbackApiTest, ContentionTest) { + auto& api = CuptiCallbackApi::singleton(); + const CUpti_CallbackDomain domain = CUPTI_CB_DOMAIN_RUNTIME_API; + const CUpti_CallbackId cbid = CUPTI_RUNTIME_TRACE_CBID_cudaLaunchKernel_v7000; + const CuptiCallbackApi::CuptiCallBackID kineto_cbid = + CuptiCallbackApi::CUDA_LAUNCH_KERNEL; + + bool ret = api.registerCallback(domain, kineto_cbid, empty_cb); + EXPECT_TRUE(ret) << "Failed to add callback"; + + const int iters = 10000; + const int num_readers = 8; + + simple_cb_calls = 0; + + // simulate callbacks being executed on multiple threads in parallel + // during this interval add a new atomic_callback. + // this test ensured mutual exclusion is working fine + auto read_fn = [&](int tid){ + auto start_ts = high_resolution_clock::now(); + for (int i = 0; i < iters; i++) { + api.__callback_switchboard(domain, cbid, nullptr); + } + auto runtime_ms = duration_cast( + high_resolution_clock::now() - start_ts); + LOG(INFO) << "th " << tid << " done in " << runtime_ms.count() << " ms"; + }; + + + std::vector read_ths; + for (int i = 0; i< num_readers; i++) { + read_ths.emplace_back(read_fn, i); + } + + ret = api.registerCallback(domain, kineto_cbid, atomic_cb); + EXPECT_TRUE(ret) << "Failed to add callback"; + + for (auto& t : read_ths) { + t.join(); + } + + //EXPECT_GT(simple_cb_calls, 0) + // << "Atomic callback should have been called at least once."; + + api.deleteCallback(domain, kineto_cbid, empty_cb); + api.deleteCallback(domain, kineto_cbid, atomic_cb); +} + +TEST(CuptiCallbackApiTest, Bechmark) { + + constexpr int iters = 1000; + // atomic bench a number of times to get a baseline + + const CUpti_CallbackDomain domain = CUPTI_CB_DOMAIN_RUNTIME_API; + const CUpti_CallbackId cbid = CUPTI_RUNTIME_TRACE_CBID_cudaLaunchKernel_v7000; + const CuptiCallbackApi::CuptiCallBackID kineto_cbid = + CuptiCallbackApi::CUDA_LAUNCH_KERNEL; + + LOG(INFO) << "Iteration count = " << iters; + + const bool use_empty = true; + auto cbfn = use_empty ? &empty_cb : &atomic_cb; + + // warmup + for (int i = 0; i < 50; i++) { + (*cbfn)(domain, cbid, nullptr); + } + + auto start_ts = high_resolution_clock::now(); + for (int i = 0; i < iters; i++) { + (*cbfn)(domain, cbid, nullptr); + } + auto delta_baseline_ns = duration_cast( + high_resolution_clock::now() - start_ts); + LOG(INFO) << "Baseline runtime = " << delta_baseline_ns.count() << " ns"; + + + auto& api = CuptiCallbackApi::singleton(); + bool ret = api.registerCallback(domain, kineto_cbid, cbfn); + EXPECT_TRUE(ret) << "Failed to add callback"; + + // warmup + for (int i = 0; i < 50; i++) { + api.__callback_switchboard(domain, cbid, nullptr); + } + + start_ts = high_resolution_clock::now(); + for (int i = 0; i < iters; i++) { + api.__callback_switchboard(domain, cbid, nullptr); + } + + auto delta_callback_ns = duration_cast( + high_resolution_clock::now() - start_ts); + LOG(INFO) << "Callback runtime = " << delta_callback_ns.count() << " ns"; + + LOG(INFO) << "Callback runtime per iteration = " << + (delta_callback_ns.count() - delta_baseline_ns.count()) / (double) iters + << " ns"; + +} diff --git a/tb_plugins/profiling/libkineto/test/CuptiProfilerApiTest.cu b/tb_plugins/profiling/libkineto/test/CuptiProfilerApiTest.cu new file mode 100644 index 000000000..54ad51b0a --- /dev/null +++ b/tb_plugins/profiling/libkineto/test/CuptiProfilerApiTest.cu @@ -0,0 +1,353 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#include +#include +#include + +#include + +// TODO(T90238193) +// @lint-ignore-every CLANGTIDY facebook-hte-RelativeInclude +#include "src/Logger.h" +#include "src/CuptiRangeProfilerApi.h" + +#define DRIVER_API_CALL(apiFuncCall) \ + do { \ + CUresult _status = apiFuncCall; \ + if (_status != CUDA_SUCCESS) { \ + LOG(ERROR) << "Failed invoking CUDA driver function " \ + << #apiFuncCall << " status = " \ + << _status; \ + exit(-1); \ + } \ + } while (0) + +#define EXPECT(expr)\ + if (!(expr)) {\ + }; + +using namespace KINETO_NAMESPACE; + +static int numRanges = 1; + +using Type = double; + +// Device code +__global__ void VecAdd(const Type* A, const Type* B, Type* C, int N) { + int i = blockDim.x * blockIdx.x + threadIdx.x; + if (i < N) { + C[i] = A[i] + B[i]; + } +} + +// Device code +__global__ void VecSub(const Type* A, const Type* B, Type* C, int N) { + int i = blockDim.x * blockIdx.x + threadIdx.x; + if (i < N) { + C[i] = A[i] - B[i]; + } +} + +static void initVec(Type* vec, int n) { + for (int i = 0; i < n; i++) { + vec[i] = i; + } +} + +static void cleanUp( + Type* h_A, + Type* h_B, + Type* h_C, + Type* h_D, + Type* d_A, + Type* d_B, + Type* d_C, + Type* d_D) { + if (d_A) + cudaFree(d_A); + if (d_B) + cudaFree(d_B); + if (d_C) + cudaFree(d_C); + if (d_D) + cudaFree(d_D); + + // Free host memory + if (h_A) + free(h_A); + if (h_B) + free(h_B); + if (h_C) + free(h_C); + if (h_D) + free(h_D); +} + +/* Benchmark application used to test profiler measurements + * This simply runs two kernels vector Add and Vector Subtract + */ + +void VectorAddSubtract() { + int N = 50000; + size_t size = N * sizeof(Type); + int threadsPerBlock = 0; + int blocksPerGrid = 0; + Type *h_A, *h_B, *h_C, *h_D; + Type *d_A, *d_B, *d_C, *d_D; + int i; + Type sum, diff; + + // Allocate input vectors h_A and h_B in host memory + h_A = (Type*)malloc(size); + h_B = (Type*)malloc(size); + h_C = (Type*)malloc(size); + h_D = (Type*)malloc(size); + + // Initialize input vectors + initVec(h_A, N); + initVec(h_B, N); + memset(h_C, 0, size); + memset(h_D, 0, size); + + // Allocate vectors in device memory + cudaMalloc((void**)&d_A, size); + cudaMalloc((void**)&d_B, size); + cudaMalloc((void**)&d_C, size); + cudaMalloc((void**)&d_D, size); + + // Copy vectors from host memory to device memory + cudaMemcpy(d_A, h_A, size, cudaMemcpyHostToDevice); + cudaMemcpy(d_B, h_B, size, cudaMemcpyHostToDevice); + + // Invoke kernel + threadsPerBlock = 256; + blocksPerGrid = (N + threadsPerBlock - 1) / threadsPerBlock; + LOG(INFO) << fmt::format( + "Launching kernel: blocks {}, thread/block {}", + blocksPerGrid, + threadsPerBlock); + + VecAdd<<>>(d_A, d_B, d_C, N); + + VecSub<<>>(d_A, d_B, d_D, N); + + // Copy result from device memory to host memory + // h_C contains the result in host memory + cudaMemcpy(h_C, d_C, size, cudaMemcpyDeviceToHost); + cudaMemcpy(h_D, d_D, size, cudaMemcpyDeviceToHost); + + // Verify result + for (i = 0; i < N; ++i) { + sum = h_A[i] + h_B[i]; + diff = h_A[i] - h_B[i]; + if (h_C[i] != sum || h_D[i] != diff) { + LOG(ERROR) << "Result verification failed"; + break; + } + } + + cleanUp(h_A, h_B, h_C, h_D, d_A, d_B, d_C, d_D); +} + +#if HAS_CUPTI_RANGE_PROFILER +bool runTestWithAutoRange( + int deviceNum, + const std::vector& metricNames, + CUcontext cuContext, + bool async) { + + // create a CUPTI range based profiling profiler + // this configures the counter data as well + CuptiRBProfilerSession profiler( + metricNames, deviceNum, 2, 1, async ? nullptr : cuContext); + + CUpti_ProfilerRange profilerRange = CUPTI_AutoRange; + CUpti_ProfilerReplayMode profilerReplayMode = CUPTI_KernelReplay; + + if (async) { + profiler.asyncStartAndEnable(profilerRange, profilerReplayMode); + } else { + profiler.start(profilerRange, profilerReplayMode); + profiler.enable(); + } + + VectorAddSubtract(); + + if (!async) { + profiler.disable(); + // stop profiler + profiler.stop(); + } else { + profiler.asyncDisableAndStop(); + } + + auto result = profiler.evaluateMetrics(true); + + // check results + EXPECT_EQ(result.metricNames.size(), 3); + EXPECT_EQ(result.rangeVals.size(), 2); + + for (const auto& measurement : result.rangeVals) { + EXPECT_EQ(measurement.values.size(), 3); + + if (measurement.values.size() == 3) { + // smsp__warps_launched.avg + EXPECT_NE(measurement.values[0], 0); + // smsp__sass_thread_inst_executed_op_dadd_pred_on.sum + // each kernel has 50000 dadd ops + EXPECT_EQ(measurement.values[1], 50000); + // sm__inst_executed_pipe_tensor.sum + //EXPECT_EQ(measurement.values[2], 0); + } + } + return true; +} + +bool runTestWithUserRange( + int deviceNum, + const std::vector& metricNames, + CUcontext cuContext, + bool async = false) { + + // create a CUPTI range based profiling profiler + // this configures the counter data as well + CuptiRBProfilerSession profiler( + metricNames, deviceNum, numRanges, 1, async ? nullptr : cuContext); + + CUpti_ProfilerRange profilerRange = CUPTI_UserRange; + CUpti_ProfilerReplayMode profilerReplayMode = CUPTI_UserReplay; + + if (async) { + profiler.asyncStartAndEnable(profilerRange, profilerReplayMode); + { VectorAddSubtract(); } + profiler.disableAndStop(); + } else { + profiler.start(profilerRange, profilerReplayMode); + + /* User takes the resposiblity of replaying the kernel launches */ + bool replay = true; + do { + profiler.beginPass(); + { + profiler.enable(); + + std::string rangeName = "vecAddSub"; + profiler.pushRange(rangeName); + + { VectorAddSubtract(); } + + profiler.popRange(); + profiler.disable(); + } + LOG(INFO) << "Replay starting."; + replay = profiler.endPass(); + + } while (!replay); + + // stop profiler + profiler.stop(); + } + VectorAddSubtract(); + auto result = profiler.evaluateMetrics(true); + + // check results + EXPECT_EQ(result.metricNames.size(), 3); + EXPECT_EQ(result.rangeVals.size(), 1); + + if (result.rangeVals.size() > 0) { + const auto& measurement = result.rangeVals[0]; + EXPECT_EQ(measurement.values.size(), 3); + + if (measurement.values.size() == 3) { + // smsp__warps_launched.avg + EXPECT_NE(measurement.values[0], 0); + // smsp__sass_thread_inst_executed_op_dadd_pred_on.sum + // in async mode multiple passes are not supported yet + if (!async) { + EXPECT_EQ(measurement.values[1], 100000); + } + // sm__inst_executed_pipe_tensor.sum + //EXPECT_EQ(measurement.values[2], 0); + } + } + return true; +} +#endif // HAS_CUPTI_RANGE_PROFILER + +int main(int argc, char* argv[]) { + + CUdevice cuDevice; + + int deviceCount, deviceNum; + int computeCapabilityMajor = 0, computeCapabilityMinor = 0; + + printf("Usage: %s [device_num]\n", argv[0]); + + DRIVER_API_CALL(cuInit(0)); + DRIVER_API_CALL(cuDeviceGetCount(&deviceCount)); + + if (deviceCount == 0) { + LOG(ERROR) << "There is no device supporting CUDA."; + return -2; + } + + if (argc > 1) + deviceNum = atoi(argv[1]); + else + deviceNum = 0; + LOG(INFO) << "CUDA Device Number: " << deviceNum; + + DRIVER_API_CALL(cuDeviceGet(&cuDevice, deviceNum)); + DRIVER_API_CALL(cuDeviceGetAttribute( + &computeCapabilityMajor, + CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, + cuDevice)); + DRIVER_API_CALL(cuDeviceGetAttribute( + &computeCapabilityMinor, + CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, + cuDevice)); + + LOG(INFO) << "Compute Cabapbility = " + << fmt::format("{},{}",computeCapabilityMajor, computeCapabilityMinor); + + if (computeCapabilityMajor < 7) { + LOG(ERROR) << "CUPTI Profiler is not supported with compute capability < 7.0"; + return -2; + } + + CuptiRBProfilerSession::staticInit(); + + // metrics to profile + std::vector metricNames = { + "smsp__warps_launched.avg", + "smsp__sass_thread_inst_executed_op_dadd_pred_on.sum", + "sm__inst_executed_pipe_tensor.sum", + }; + + CUcontext cuContext; + DRIVER_API_CALL(cuCtxCreate(&cuContext, 0, cuDevice)); + + VectorAddSubtract(); + +#if HAS_CUPTI_RANGE_PROFILER + CuptiRBProfilerSession::staticInit(); + + if (!runTestWithUserRange(deviceNum, metricNames, cuContext, false)) { + LOG(ERROR) << "Failed to profiler test benchmark in user range"; + } else if (!runTestWithAutoRange(deviceNum, metricNames, cuContext, false)) { + LOG(ERROR) << "Failed to profiler test benchmark in auto range"; + } else if (!runTestWithUserRange(deviceNum, metricNames, cuContext, true)) { + LOG(ERROR) << "Failed to profiler test benchmark in user range async"; + } else if (!runTestWithAutoRange(deviceNum, metricNames, cuContext, true)) { + LOG(ERROR) << "Failed to profiler test benchmark in auto range async"; + } + + CuptiRBProfilerSession::deInitCupti(); +#else + LOG(WARNING) << "CuptiRBProfilerSession is not supported."; +#endif // HAS_CUPTI_RANGE_PROFILER + DRIVER_API_CALL(cuCtxDestroy(cuContext)); + + + return 0; +} diff --git a/tb_plugins/profiling/libkineto/test/CuptiRangeProfilerApiTest.cpp b/tb_plugins/profiling/libkineto/test/CuptiRangeProfilerApiTest.cpp new file mode 100644 index 000000000..28cad722c --- /dev/null +++ b/tb_plugins/profiling/libkineto/test/CuptiRangeProfilerApiTest.cpp @@ -0,0 +1,113 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#include +#include +#include + +#include "include/libkineto.h" +#include "include/Config.h" +#include "src/CuptiRangeProfilerApi.h" + +#include "src/Logger.h" +#include "test/CuptiRangeProfilerTestUtil.h" + +using namespace KINETO_NAMESPACE; + +#if HAS_CUPTI_PROFILER + +TEST(CuptiRangeProfilerApiTest, contextTracking) { + std::vector log_modules( + {"CuptiRangeProfilerApi.cpp"}); + SET_LOG_VERBOSITY_LEVEL(1, log_modules); + + std::array data; + std::array contexts; + for (int i = 0; i < data.size(); i++) { + contexts[i] = reinterpret_cast(&data[i]); + } + + // simulate creating contexts, this calls the trackCudaContexts + // function that would otherwise be called via a callback + uint32_t dev = 0; + for (auto ctx : contexts) { + simulateCudaContextCreate(ctx, dev++); + } + + EXPECT_EQ( + CuptiRBProfilerSession::getActiveDevices(), + std::set({0, 1, 2})); + + simulateCudaContextDestroy(contexts[1], 1); + + EXPECT_EQ( + CuptiRBProfilerSession::getActiveDevices(), + std::set({0, 2})); + + simulateCudaContextDestroy(contexts[0], 0); + simulateCudaContextDestroy(contexts[2], 2); + + EXPECT_TRUE( + CuptiRBProfilerSession::getActiveDevices().empty()); +} + +TEST(CuptiRangeProfilerApiTest, asyncLaunchUserRange) { + std::vector log_modules( + {"CuptiRangeProfilerApi.cpp"}); + SET_LOG_VERBOSITY_LEVEL(1, log_modules); + + // this is bad but the pointer is never accessed + CUcontext ctx0 = reinterpret_cast(10); + simulateCudaContextCreate(ctx0, 0 /*device_id*/); + + auto session = std::make_unique(0, ctx0); + session->asyncStartAndEnable(CUPTI_UserRange, CUPTI_UserReplay); + + simulateKernelLaunch(ctx0, "hello"); + simulateKernelLaunch(ctx0, "foo"); + simulateKernelLaunch(ctx0, "bar"); + + session->asyncDisableAndStop(); + // stop happens after next kernel is run + simulateKernelLaunch(ctx0, "bar"); + simulateCudaContextDestroy(ctx0, 0 /*device_id*/); + + EXPECT_EQ(session->passes_ended, 1); + EXPECT_EQ(session->ranges_ended, 1); + EXPECT_TRUE(session->enabled); +} + +TEST(CuptiRangeProfilerApiTest, asyncLaunchAutoRange) { + std::vector log_modules( + {"CuptiRangeProfilerApi.cpp"}); + SET_LOG_VERBOSITY_LEVEL(1, log_modules); + + // this is bad but the pointer is never accessed + CUcontext ctx0 = reinterpret_cast(10); + CUcontext ctx1 = reinterpret_cast(11); + + simulateCudaContextCreate(ctx0, 0 /*device_id*/); + + auto session = std::make_unique(0, ctx0); + session->asyncStartAndEnable(CUPTI_AutoRange, CUPTI_KernelReplay); + + simulateKernelLaunch(ctx0, "hello"); + simulateKernelLaunch(ctx0, "foo"); + simulateKernelLaunch(ctx1, "kernel_on_different_device"); + simulateKernelLaunch(ctx0, "bar"); + + session->asyncDisableAndStop(); + // stop happens after next kernel is run + simulateKernelLaunch(ctx0, "bar"); + simulateCudaContextDestroy(ctx0, 0 /*device_id*/); + + EXPECT_EQ(session->passes_ended, 0); + EXPECT_EQ(session->ranges_ended, 0); + EXPECT_TRUE(session->enabled); + + EXPECT_EQ( + session->getKernelNames(), + std::vector({"hello", "foo", "bar"})) + << "Kernel names were not tracked"; +} + +#endif // HAS_CUPTI_PROFILER diff --git a/tb_plugins/profiling/libkineto/test/CuptiRangeProfilerConfigTest.cpp b/tb_plugins/profiling/libkineto/test/CuptiRangeProfilerConfigTest.cpp new file mode 100644 index 000000000..3f5689682 --- /dev/null +++ b/tb_plugins/profiling/libkineto/test/CuptiRangeProfilerConfigTest.cpp @@ -0,0 +1,67 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#include "include/Config.h" +#include "src/CuptiRangeProfilerConfig.h" + +#include +#include +#include +#include + +using namespace std::chrono; +using namespace KINETO_NAMESPACE; + +class CuptiRangeProfilerConfigTest : public ::testing::Test { + protected: + void SetUp() override { + CuptiRangeProfilerConfig::registerFactory(); + } +}; + +TEST_F(CuptiRangeProfilerConfigTest, ConfigureProfiler) { + Config cfg; + std::vector metrics = { + "kineto__cuda_core_flops", + "sm__inst_executed.sum", + "l1tex__data_bank_conflicts_pipe_lsu.sum", + }; + auto metricsConfigStr = + fmt::format("CUPTI_PROFILER_METRICS = {}", fmt::join(metrics, ",")); + + EXPECT_TRUE(cfg.parse(metricsConfigStr)); + EXPECT_TRUE(cfg.parse("CUPTI_PROFILER_ENABLE_PER_KERNEL = true")); + EXPECT_TRUE(cfg.parse("CUPTI_PROFILER_MAX_RANGES = 42")); + + const CuptiRangeProfilerConfig& cupti_cfg = + CuptiRangeProfilerConfig::get(cfg); + + EXPECT_EQ(cupti_cfg.activitiesCuptiMetrics(), metrics); + EXPECT_EQ(cupti_cfg.cuptiProfilerPerKernel(), true); + EXPECT_EQ(cupti_cfg.cuptiProfilerMaxRanges(), 42); + +} + +TEST_F(CuptiRangeProfilerConfigTest, RangesDefaults) { + Config cfg, cfg_auto; + + // do not set max ranges in config, check defaults are sane + EXPECT_TRUE(cfg.parse("CUPTI_PROFILER_METRICS = kineto__cuda_core_flops")); + EXPECT_TRUE(cfg.parse("CUPTI_PROFILER_ENABLE_PER_KERNEL = false")); + + cfg.setSignalDefaults(); + + EXPECT_TRUE(cfg_auto.parse("CUPTI_PROFILER_METRICS = kineto__cuda_core_flops")); + EXPECT_TRUE(cfg_auto.parse("CUPTI_PROFILER_ENABLE_PER_KERNEL = true")); + + cfg_auto.setClientDefaults(); + + int user_ranges, auto_ranges; + + user_ranges = CuptiRangeProfilerConfig::get(cfg).cuptiProfilerMaxRanges(); + auto_ranges = CuptiRangeProfilerConfig::get(cfg_auto).cuptiProfilerMaxRanges(); + + EXPECT_GE(user_ranges, 1) << " in user range mode default to at least 1 ranges"; + EXPECT_GE(auto_ranges, 1000) << " in auto range mode default to at least 1000 ranges"; + + EXPECT_GT(auto_ranges, user_ranges); +} diff --git a/tb_plugins/profiling/libkineto/test/CuptiRangeProfilerTestUtil.h b/tb_plugins/profiling/libkineto/test/CuptiRangeProfilerTestUtil.h new file mode 100644 index 000000000..861b65fd7 --- /dev/null +++ b/tb_plugins/profiling/libkineto/test/CuptiRangeProfilerTestUtil.h @@ -0,0 +1,96 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#include +#include + +// TODO(T90238193) +// @lint-ignore-every CLANGTIDY facebook-hte-RelativeInclude +#include "CuptiRangeProfilerApi.h" + +namespace KINETO_NAMESPACE { + +#if HAS_CUPTI_PROFILER + +class MockCuptiRBProfilerSession : public CuptiRBProfilerSession { + public: + MockCuptiRBProfilerSession(int deviceId, CUcontext ctx) + : CuptiRBProfilerSession(deviceId, ctx) {} + + void beginPass() override { + LOG(INFO) << " Mock CUPTI begin pass"; + passes_started++; + } + + bool endPass() override { + passes_ended++; + return true; + } + + void flushCounterData() override {} + + void pushRange(const std::string& rangeName) override { + LOG(INFO) << " Mock CUPTI pushrange ( " << rangeName << " )"; + ranges_started++; + } + + void popRange() override { + LOG(INFO) << " Mock CUPTI poprange"; + ranges_ended++; + } + + void stop() override { + runChecks(); + } + + void enable() override { + enabled = true; + } + void disable() override {} + + CuptiProfilerResult evaluateMetrics(bool /*verbose*/) override { + return result; + } + +protected: + void startInternal( + CUpti_ProfilerRange profilerRange, + CUpti_ProfilerReplayMode profilerReplayMode) override { + curRange_ = profilerRange; + curReplay_ = profilerReplayMode; + } + +private: + void runChecks() { + EXPECT_EQ(passes_started, passes_ended); + EXPECT_EQ(ranges_started, ranges_ended); + } + + public: + int passes_started = 0; + int passes_ended = 0; + int ranges_started = 0; + int ranges_ended = 0; + bool enabled = false; + + CuptiProfilerResult result; + +}; + +inline void simulateCudaContextCreate(CUcontext context, uint32_t dev) { + testing::trackCudaCtx( + context, dev, CUPTI_CBID_RESOURCE_CONTEXT_CREATED); +} + +inline void simulateCudaContextDestroy(CUcontext context, uint32_t dev) { + testing::trackCudaCtx( + context, dev, CUPTI_CBID_RESOURCE_CONTEXT_DESTROY_STARTING); +} + +inline void simulateKernelLaunch( + CUcontext context, const std::string& kernelName) { + testing::trackCudaKernelLaunch(context, kernelName.c_str()); +} + +#endif // HAS_CUPTI_PROFILER + +} // namespace KINETO_NAMESPACE diff --git a/tb_plugins/profiling/libkineto/test/CuptiStringsTest.cpp b/tb_plugins/profiling/libkineto/test/CuptiStringsTest.cpp new file mode 100644 index 000000000..405f9404a --- /dev/null +++ b/tb_plugins/profiling/libkineto/test/CuptiStringsTest.cpp @@ -0,0 +1,29 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#include + +#include "src/cupti_strings.h" + +using namespace KINETO_NAMESPACE; + +TEST(CuptiStringsTest, Valid) { + ASSERT_STREQ( + runtimeCbidName(CUPTI_RUNTIME_TRACE_CBID_INVALID), "INVALID"); + ASSERT_STREQ( + runtimeCbidName(CUPTI_RUNTIME_TRACE_CBID_cudaDriverGetVersion_v3020), + "cudaDriverGetVersion"); + ASSERT_STREQ(runtimeCbidName + (CUPTI_RUNTIME_TRACE_CBID_cudaDeviceSynchronize_v3020), + "cudaDeviceSynchronize"); + ASSERT_STREQ( + runtimeCbidName(CUPTI_RUNTIME_TRACE_CBID_cudaStreamSetAttribute_ptsz_v11000), + "cudaStreamSetAttribute_ptsz"); +} + +TEST(CuptiStringsTest, Invalid) { + ASSERT_STREQ(runtimeCbidName(-1), "INVALID"); + // We can't actually use CUPTI_RUNTIME_TRACE_CBID_SIZE here until we + // auto-generate the string table, since it may have more entries than + // the enum in the version used to compile. + ASSERT_STREQ(runtimeCbidName(1000), "INVALID"); +} diff --git a/tb_plugins/profiling/libkineto/test/EventProfilerTest.cpp b/tb_plugins/profiling/libkineto/test/EventProfilerTest.cpp new file mode 100644 index 000000000..cb36c826a --- /dev/null +++ b/tb_plugins/profiling/libkineto/test/EventProfilerTest.cpp @@ -0,0 +1,578 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#include "src/EventProfiler.h" + +#include +#include +#include + +using namespace std::chrono; +using namespace KINETO_NAMESPACE; + +TEST(PercentileTest, Create) { + PercentileList pct = {{10, SampleValue(0)}, + {49, SampleValue(0)}, + {50, SampleValue(0)}, + {90, SampleValue(0)}}; + + percentiles({0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100}, pct); + EXPECT_EQ(pct[0].second.getInt(), 10); + EXPECT_EQ(pct[1].second.getInt(), 50); + EXPECT_EQ(pct[2].second.getInt(), 50); + EXPECT_EQ(pct[3].second.getInt(), 90); + + percentiles({80, 10, 20, 70, 60, 40, 90, 30, 50, 0, 100}, pct); + EXPECT_EQ(pct[0].second.getInt(), 10); + EXPECT_EQ(pct[1].second.getInt(), 50); + EXPECT_EQ(pct[2].second.getInt(), 50); + EXPECT_EQ(pct[3].second.getInt(), 90); + + percentiles({80}, pct); + EXPECT_EQ(pct[0].second.getInt(), 80); + EXPECT_EQ(pct[1].second.getInt(), 80); + EXPECT_EQ(pct[2].second.getInt(), 80); + EXPECT_EQ(pct[3].second.getInt(), 80); + + percentiles({80, 50}, pct); + EXPECT_EQ(pct[0].second.getInt(), 50); + EXPECT_EQ(pct[1].second.getInt(), 50); + EXPECT_EQ(pct[2].second.getInt(), 80); + EXPECT_EQ(pct[3].second.getInt(), 80); +} + +TEST(PercentileTest, Normalize) { + PercentileList pct = { + {10, SampleValue(10)}, {50, SampleValue(100.0)}, {90, SampleValue(2000)}}; + + normalize(pct, 2.5); + + EXPECT_EQ(pct[0].second.getInt(), 25); + EXPECT_EQ((int)pct[1].second.getDouble(), 250); + EXPECT_EQ(pct[2].second.getInt(), 5000); +} + +TEST(EventTest, SumSamples) { + Event ev; + ev.instanceCount = 4; + auto t = system_clock::now(); + ev.addSample(t, {1, 2, 3, 4}); + ev.addSample(t, {10, 20, 30, 40}); + ev.addSample(t, {100, 200, 300, 400}); + + EXPECT_EQ(ev.sumInstance(0, {0, 0, 3}), 1); + EXPECT_EQ(ev.sumInstance(0, {0, 1, 3}), 10); + EXPECT_EQ(ev.sumInstance(0, {0, 2, 3}), 100); + + EXPECT_EQ(ev.sumInstance(0, {0, 0, 1}), 111); + + EXPECT_EQ(ev.sumInstance(3, {0, 0, 1}), 444); + + // Non-zero offset + EXPECT_EQ(ev.sumInstance(0, {1, 0, 2}), 10); + EXPECT_EQ(ev.sumInstance(0, {1, 1, 2}), 100); + EXPECT_EQ(ev.sumInstance(0, {1, 0, 1}), 110); + + ev.addSample(t, {1000, 2000, 3000, 4000}); + + EXPECT_EQ(ev.sumInstance(0, {1, 0, 3}), 10); + EXPECT_EQ(ev.sumInstance(0, {1, 1, 3}), 100); + EXPECT_EQ(ev.sumInstance(0, {2, 1, 2}), 1000); + EXPECT_EQ(ev.sumInstance(0, {2, 0, 1}), 1100); + + EXPECT_EQ(ev.sumAll({0, 0, 4}), 10); + EXPECT_EQ(ev.sumAll({1, 0, 3}), 100); + EXPECT_EQ(ev.sumAll({2, 1, 2}), 10000); + EXPECT_EQ(ev.sumAll({0, 1, 2}), 11000); + EXPECT_EQ(ev.sumAll({0, 0, 1}), 11110); +} + +TEST(EventTest, Percentiles) { + Event ev; + ev.instanceCount = 4; + auto t = system_clock::now(); + ev.addSample(t, {3, 2, 1, 4}); + ev.addSample(t, {30, 20, 10, 40}); + ev.addSample(t, {300, 200, 100, 400}); + + PercentileList pct = { + {10, SampleValue(0)}, {50, SampleValue(0)}, {90, SampleValue(0)}}; + + ev.percentiles(pct, {0, 0, 3}); + EXPECT_EQ(pct[0].second.getInt(), 1); + EXPECT_EQ(pct[1].second.getInt(), 3); + EXPECT_EQ(pct[2].second.getInt(), 4); + + ev.percentiles(pct, {0, 0, 1}); + EXPECT_EQ(pct[0].second.getInt(), 111); + EXPECT_EQ(pct[1].second.getInt(), 333); + EXPECT_EQ(pct[2].second.getInt(), 444); +} + +class MockCuptiMetrics : public CuptiMetricApi { + public: + MockCuptiMetrics() : CuptiMetricApi(0) {} + MOCK_METHOD1(idFromName, CUpti_MetricID(const std::string& name)); + MOCK_METHOD1( + events, + std::map(CUpti_MetricID metric_id)); + MOCK_METHOD1(valueKind, CUpti_MetricValueKind(CUpti_MetricID metric)); + MOCK_METHOD1( + evaluationMode, + CUpti_MetricEvaluationMode(CUpti_MetricID metric)); + MOCK_METHOD5( + calculate, + SampleValue( + CUpti_MetricID metric, + CUpti_MetricValueKind kind, + std::vector& events, + std::vector& values, + int64_t duration)); +}; + +TEST(MetricTest, Calculate) { + using ::testing::Return; + MockCuptiMetrics metrics; + + // The events used for the ipc metrics: instructions and cycles + // Pretend we have 2 SMs and 2 samples of each event + Event instr("instructions"); + instr.instanceCount = 2; + auto t = system_clock::now(); + instr.addSample(t, {100, 200}); + instr.addSample(t, {300, 400}); + + Event cycles("cycles"); + cycles.instanceCount = 2; + cycles.addSample(t, {1000, 1200}); + cycles.addSample(t, {1300, 1300}); + + // 2 & 3 are the event ids we specified in the metric + std::map events; + events[2] = std::move(instr); + events[3] = std::move(cycles); + + // Define an ipc metric + EXPECT_CALL(metrics, valueKind(1)) + .Times(1) + .WillOnce(Return(CUPTI_METRIC_VALUE_KIND_DOUBLE)); + Metric m( + "ipc", 1, {2, 3}, CUPTI_METRIC_EVALUATION_MODE_PER_INSTANCE, metrics); + + // Calculate metric for first sample + // Since evaluation mode is CUPTI_METRIC_EVALUATION_MODE_PER_INSTANCE, + // Cupti API will be called three times: once for each SM (2) and once + // to get the total across SMs. + std::vector ids = {2, 3}; + std::vector vals = {100, 1000}; + EXPECT_CALL( + metrics, calculate(1, CUPTI_METRIC_VALUE_KIND_DOUBLE, ids, vals, 1000)) + .Times(1) + .WillOnce(Return(SampleValue(0.1))); + vals = {200, 1200}; + EXPECT_CALL( + metrics, calculate(1, CUPTI_METRIC_VALUE_KIND_DOUBLE, ids, vals, 1000)) + .Times(1) + .WillOnce(Return(SampleValue(0.17))); + vals = {300, 2200}; + EXPECT_CALL( + metrics, calculate(1, CUPTI_METRIC_VALUE_KIND_DOUBLE, ids, vals, 1000)) + .Times(1) + .WillOnce(Return(SampleValue(0.14))); + auto v = m.calculate(events, nanoseconds(1000), {0, 0, 2}); + + EXPECT_EQ(v.perInstance.size(), 2); + EXPECT_EQ(v.perInstance[0].getDouble(), 0.1); + EXPECT_EQ(v.perInstance[1].getDouble(), 0.17); + EXPECT_EQ(v.total.getDouble(), 0.14); + + // Calculate second sample. + // Change evaluation mode to CUPTI_METRIC_EVALUATION_MODE_AGGREGATE. + // Now we should get only one call to the Cupti API for the total. + EXPECT_CALL(metrics, valueKind(1)) + .Times(1) + .WillOnce(Return(CUPTI_METRIC_VALUE_KIND_DOUBLE)); + Metric m2("ipc", 1, {2, 3}, CUPTI_METRIC_EVALUATION_MODE_AGGREGATE, metrics); + vals = {700, 2600}; + EXPECT_CALL( + metrics, calculate(1, CUPTI_METRIC_VALUE_KIND_DOUBLE, ids, vals, 1000)) + .Times(1) + .WillOnce(Return(SampleValue(0.27))); + v = m2.calculate(events, nanoseconds(1000), {0, 1, 2}); + + EXPECT_EQ(v.perInstance.size(), 1); + EXPECT_EQ(v.perInstance[0].getDouble(), 0.27); + EXPECT_EQ(v.total.getDouble(), 0.27); +} + +class MockCuptiEvents : public CuptiEventApi { + public: + MOCK_METHOD1( + createGroupSets, + CUpti_EventGroupSets*(std::vector& ids)); + MOCK_METHOD1(destroyGroupSets, void(CUpti_EventGroupSets* sets)); + MOCK_METHOD0(setContinuousMode, bool()); + MOCK_METHOD1(enablePerInstance, void(CUpti_EventGroup eventGroup)); + MOCK_METHOD1(instanceCount, uint32_t(CUpti_EventGroup eventGroup)); + MOCK_METHOD1(enableGroupSet, void(CUpti_EventGroupSet& set)); + MOCK_METHOD1(disableGroupSet, void(CUpti_EventGroupSet& set)); + MOCK_METHOD3( + readEvent, + void(CUpti_EventGroup g, CUpti_EventID id, std::vector& vals)); + MOCK_METHOD1(eventsInGroup, std::vector(CUpti_EventGroup g)); + MOCK_METHOD1(eventId, CUpti_EventID(const std::string& name)); +}; + +TEST(EventGroupSetTest, CollectSample) { + using ::testing::_; + using ::testing::Return; + using ::testing::SetArgPointee; + const CUpti_EventGroup g1{nullptr}; + const CUpti_EventGroup g2{reinterpret_cast(0x1000)}; + CUpti_EventGroup groups[] = {g1, g2}; + CUpti_EventGroupSet set; + set.eventGroups = groups; + set.numEventGroups = 2; + + std::map events; + Event instr("instructions"); + events[4] = std::move(instr); + Event cycles("cycles"); + events[5] = std::move(cycles); + Event branches("branches"); + events[10] = std::move(branches); + + MockCuptiEvents cupti_events; + EXPECT_CALL(cupti_events, enablePerInstance(g1)).Times(1); + EXPECT_CALL(cupti_events, enablePerInstance(g2)).Times(1); + EXPECT_CALL(cupti_events, instanceCount(g1)).Times(1).WillOnce(Return(80)); + EXPECT_CALL(cupti_events, instanceCount(g2)).Times(1).WillOnce(Return(40)); + std::vector events_in_group1 = {4, 5}; + EXPECT_CALL(cupti_events, eventsInGroup(g1)) + .Times(1) + .WillOnce(Return(events_in_group1)); + std::vector events_in_group2 = {10}; + EXPECT_CALL(cupti_events, eventsInGroup(g2)) + .Times(1) + .WillOnce(Return(events_in_group2)); + EventGroupSet group_set(set, events, cupti_events); + + EXPECT_EQ(group_set.groupCount(), 2); + EXPECT_EQ(events[4].instanceCount, 80); + EXPECT_EQ(events[5].instanceCount, 80); + EXPECT_EQ(events[10].instanceCount, 40); + + // This should not cause any Cupti API action as the group + // set is already disabled + group_set.setEnabled(false); + + // Activate group set - if activated twice, only the first + // should cause cupti API to be called + EXPECT_CALL(cupti_events, enableGroupSet(_)).Times(1); + group_set.setEnabled(false); + group_set.setEnabled(true); + + EXPECT_CALL(cupti_events, eventsInGroup(g1)) + .Times(1) + .WillOnce(Return(events_in_group1)); + EXPECT_CALL(cupti_events, eventsInGroup(g2)) + .Times(1) + .WillOnce(Return(events_in_group2)); + EXPECT_CALL(cupti_events, readEvent(g1, 4, _)).Times(1); + EXPECT_CALL(cupti_events, readEvent(g1, 5, _)).Times(1); + EXPECT_CALL(cupti_events, readEvent(g2, 10, _)).Times(1); + group_set.collectSample(); + + EXPECT_EQ(events[4].sampleCount(), 1); + EXPECT_EQ(events[5].sampleCount(), 1); + EXPECT_EQ(events[10].sampleCount(), 1); +} + +class MockLogger : public SampleListener { + public: + MOCK_METHOD3(handleSample, void(int device, const Sample& sample, bool from_new_version)); + MOCK_METHOD1(update, void(const Config& config)); +}; + +class EventProfilerTest : public ::testing::Test { + protected: + void SetUp() override { + auto cupti_events_ptr = std::make_unique(); + auto cupti_metrics_ptr = std::make_unique(); + cuptiEvents_ = cupti_events_ptr.get(); + cuptiMetrics_ = cupti_metrics_ptr.get(); + loggers_.push_back(std::make_unique()); + onDemandLoggers_.push_back(std::make_unique()); + profiler_ = std::make_unique( + std::move(cupti_events_ptr), + std::move(cupti_metrics_ptr), + loggers_, + onDemandLoggers_); + + for (int i = 0; i < kEventGroupCount; i++) { + eventGroups_[i] = &eventGroups_[i]; + } + for (int i = 0; i < kGroupSetCount; i++) { + // Default size to 1 but can be changed by test + groupSet_[i].numEventGroups = 1; + // Two groups per set + groupSet_[i].eventGroups = &eventGroups_[i * 2]; + } + groupSets_.numSets = 1; + groupSets_.sets = groupSet_; + } + + MockCuptiEvents* cuptiEvents_; + MockCuptiMetrics* cuptiMetrics_; + std::vector> loggers_; + std::vector> onDemandLoggers_; + constexpr static int kEventGroupCount = 4; + constexpr static int kGroupSetCount = 2; + CUpti_EventGroup eventGroups_[kEventGroupCount]; + CUpti_EventGroupSet groupSet_[kGroupSetCount]; + CUpti_EventGroupSets groupSets_; + std::unique_ptr profiler_; +}; + +TEST_F(EventProfilerTest, ConfigureFailure) { + using namespace testing; + + // Default config has no counters enabled. + // Check that profiler remains disabled. + Config cfg; + profiler_->configure(cfg, nullptr); + + EXPECT_FALSE(profiler_->enabled()); + + // There is no event named "cycles" + // In this case the profiler should print a warning and remain disabled + bool parsed = cfg.parse("EVENTS = cycles"); + EXPECT_TRUE(parsed); + + // EventProfiler should handle exception thrown from createGroupSets + // Configuration will be applied twice - once for combined base + on-demand + // and then again falling back to base + EXPECT_CALL(*cuptiEvents_, eventId("cycles")) + .Times(2) + .WillRepeatedly(Return(0)); + std::vector ids = {0}; + EXPECT_CALL(*cuptiEvents_, createGroupSets(ids)) + .Times(2) + .WillRepeatedly(Throw( + std::system_error(EINVAL, std::generic_category(), "Event ID"))); + profiler_->configure(cfg, nullptr); + + EXPECT_FALSE(profiler_->enabled()); +} + +TEST_F(EventProfilerTest, ConfigureBase) { + using namespace testing; + + // Test normal path, simple base config + Config cfg; + bool parsed = cfg.parse("EVENTS = elapsed_cycles_sm"); + EXPECT_TRUE(parsed); + + // One valid event - expect one call to eventId and createGroupSets + EXPECT_CALL(*cuptiEvents_, eventId("elapsed_cycles_sm")) + .Times(1) + .WillOnce(Return(5)); + std::vector ids = {5}; + EXPECT_CALL(*cuptiEvents_, createGroupSets(ids)) + .Times(1) + .WillOnce(Return(&groupSets_)); + EXPECT_CALL(*cuptiEvents_, enablePerInstance(eventGroups_[0])).Times(1); + EXPECT_CALL(*cuptiEvents_, instanceCount(eventGroups_[0])) + .Times(1) + .WillOnce(Return(80)); + EXPECT_CALL(*cuptiEvents_, eventsInGroup(eventGroups_[0])) + .Times(1) + .WillOnce(Return(ids)); + EXPECT_CALL(*cuptiEvents_, enableGroupSet(_)).Times(1); + + profiler_->configure(cfg, nullptr); + + EXPECT_TRUE(profiler_->enabled()); +} + +TEST_F(EventProfilerTest, ConfigureOnDemand) { + using namespace testing; + + // Test base + on-demand config, one event and one metric + Config cfg, on_demand_cfg; + bool parsed = cfg.parse(R"( + EVENTS = active_cycles + SAMPLE_PERIOD_MSECS=500 + REPORT_PERIOD_SECS=10 + SAMPLES_PER_REPORT=5 + )"); + EXPECT_TRUE(parsed); + + parsed = on_demand_cfg.parse(R"( + METRICS = ipc + EVENTS_DURATION_SECS=60 + SAMPLE_PERIOD_MSECS=200 + MULTIPLEX_PERIOD_MSECS=2000 + REPORT_PERIOD_SECS=3 + SAMPLES_PER_REPORT=10 + )"); + EXPECT_TRUE(parsed); + + // One event + EXPECT_CALL(*cuptiEvents_, eventId("active_cycles")) + .Times(1) + .WillOnce(Return(3)); + // One metric + EXPECT_CALL(*cuptiMetrics_, idFromName("ipc")).Times(1).WillOnce(Return(10)); + std::map ipc_events; + ipc_events[4] = "instructions"; + ipc_events[5] = "elapsed_cycles_sm"; + EXPECT_CALL(*cuptiMetrics_, events(10)).Times(1).WillOnce(Return(ipc_events)); + EXPECT_CALL(*cuptiMetrics_, evaluationMode(10)) + .Times(1) + .WillOnce(Return(CUPTI_METRIC_EVALUATION_MODE_PER_INSTANCE)); + EXPECT_CALL(*cuptiMetrics_, valueKind(10)) + .Times(1) + .WillOnce(Return(CUPTI_METRIC_VALUE_KIND_DOUBLE)); + std::vector ids = {3, 4, 5}; + groupSet_[0].numEventGroups = 2; + groupSets_.numSets = 2; + EXPECT_CALL(*cuptiEvents_, createGroupSets(ids)) + .Times(1) + .WillOnce(Return(&groupSets_)); + // Specified CUPTI_METRIC_EVALUATION_MODE_PER_INSTANCE per instance above + // So check that it's enabled + EXPECT_CALL(*cuptiEvents_, enablePerInstance(eventGroups_[0])).Times(1); + EXPECT_CALL(*cuptiEvents_, enablePerInstance(eventGroups_[1])).Times(1); + EXPECT_CALL(*cuptiEvents_, enablePerInstance(eventGroups_[2])).Times(1); + std::vector ids_g1{3}, ids_g2{4}, ids_g3{5}; + EXPECT_CALL(*cuptiEvents_, eventsInGroup(eventGroups_[0])) + .Times(1) + .WillOnce(Return(ids_g1)); + EXPECT_CALL(*cuptiEvents_, eventsInGroup(eventGroups_[1])) + .Times(1) + .WillOnce(Return(ids_g2)); + EXPECT_CALL(*cuptiEvents_, eventsInGroup(eventGroups_[2])) + .Times(1) + .WillOnce(Return(ids_g3)); + EXPECT_CALL(*cuptiEvents_, enableGroupSet(_)).Times(1); + + profiler_->configure(cfg, &on_demand_cfg); + + EXPECT_TRUE(profiler_->enabled()); + EXPECT_EQ(profiler_->samplePeriod().count(), 250); + EXPECT_EQ(profiler_->multiplexPeriod().count(), 1000); + EXPECT_EQ(profiler_->reportPeriod().count(), 10000); + EXPECT_EQ(profiler_->onDemandReportPeriod().count(), 4000); +} + +TEST_F(EventProfilerTest, ReportSample) { + using namespace testing; + + // Test base + on-demand config, one event and one metric + Config cfg, on_demand_cfg; + bool parsed = cfg.parse("EVENTS = active_cycles"); + EXPECT_TRUE(parsed); + + parsed = on_demand_cfg.parse(R"( + METRICS = ipc + EVENTS_DURATION_SECS=60 + )"); + EXPECT_TRUE(parsed); + + // One event + EXPECT_CALL(*cuptiEvents_, eventId("active_cycles")) + .Times(1) + .WillOnce(Return(3)); + // One metric + EXPECT_CALL(*cuptiMetrics_, idFromName("ipc")).Times(1).WillOnce(Return(10)); + std::map ipc_events; + ipc_events[4] = "instructions"; + ipc_events[5] = "elapsed_cycles_sm"; + EXPECT_CALL(*cuptiMetrics_, events(10)).Times(1).WillOnce(Return(ipc_events)); + EXPECT_CALL(*cuptiMetrics_, evaluationMode(10)) + .Times(1) + .WillOnce(Return(CUPTI_METRIC_EVALUATION_MODE_PER_INSTANCE)); + EXPECT_CALL(*cuptiMetrics_, valueKind(10)) + .Times(1) + .WillOnce(Return(CUPTI_METRIC_VALUE_KIND_DOUBLE)); + std::vector ids = {3, 4, 5}; + groupSet_[0].numEventGroups = 2; + groupSets_.numSets = 2; + EXPECT_CALL(*cuptiEvents_, createGroupSets(ids)) + .Times(1) + .WillOnce(Return(&groupSets_)); + EXPECT_CALL(*cuptiEvents_, instanceCount(_)) + .Times(3) + .WillRepeatedly(Return(4)); + std::vector ids_g1{3}, ids_g2{4}, ids_g3{5}; + // These will be called by collectSample() as well, which is called twice + // per group set + EXPECT_CALL(*cuptiEvents_, eventsInGroup(eventGroups_[0])) + .Times(3) + .WillRepeatedly(Return(ids_g1)); + EXPECT_CALL(*cuptiEvents_, eventsInGroup(eventGroups_[1])) + .Times(3) + .WillRepeatedly(Return(ids_g2)); + EXPECT_CALL(*cuptiEvents_, eventsInGroup(eventGroups_[2])) + .Times(3) + .WillRepeatedly(Return(ids_g3)); + EXPECT_CALL(*cuptiEvents_, enableGroupSet(_)).Times(1); + + profiler_->configure(cfg, &on_demand_cfg); + + EXPECT_TRUE(profiler_->enabled()); + + EXPECT_CALL(*cuptiEvents_, readEvent(_, _, _)) + .Times(6) + .WillRepeatedly(Invoke( + [](CUpti_EventGroup g, CUpti_EventID id, std::vector& vals) { + vals = {1, 2, 3, 4}; + })); + + // Need to collect four times - twice for each group set + profiler_->collectSample(); + profiler_->collectSample(); + EXPECT_CALL(*cuptiEvents_, disableGroupSet(_)).Times(1); + EXPECT_CALL(*cuptiEvents_, enableGroupSet(_)).Times(1); + profiler_->enableNextCounterSet(); + profiler_->collectSample(); + profiler_->collectSample(); + + std::vector ipc_ids = {4, 5}; + // Called once for each instance (4) and once for the total. + // x2 since we recompute per logger. + EXPECT_CALL( + *cuptiMetrics_, + calculate(10, CUPTI_METRIC_VALUE_KIND_DOUBLE, ipc_ids, _, 2000000000)) + .Times(10) + .WillRepeatedly(Return(SampleValue(0.3))); + auto& logger = dynamic_cast(*loggers_[0]); + EXPECT_CALL(logger, handleSample(0, _, _)) + .Times(1) + .WillOnce(Invoke([](int device, const Sample& sample, bool from_new_version) { + // Sample will include all stats - logger must pick the + // ones it wants. + EXPECT_EQ(sample.stats.size(), 4); + EXPECT_EQ(sample.stats[0].name, "active_cycles"); + EXPECT_EQ(sample.stats[1].name, "instructions"); + EXPECT_EQ(sample.stats[2].name, "elapsed_cycles_sm"); + EXPECT_EQ(sample.stats[3].name, "ipc"); + // 2 samples, each with values {1, 2, 3, 4} + // i.e. {2, 4, 6, 8} total + EXPECT_EQ(sample.stats[0].total.getInt(), 20); + EXPECT_EQ(sample.stats[0].percentileValues[0].second.getInt(), 2); + EXPECT_EQ(sample.stats[0].percentileValues.back().second.getInt(), 8); + // ipc is always 0.3 from mocked calculate function above + EXPECT_EQ(sample.stats[3].total.getDouble(), 0.3); + EXPECT_EQ(sample.stats[3].percentileValues[0].second.getDouble(), 0.3); + EXPECT_EQ( + sample.stats[3].percentileValues.back().second.getDouble(), 0.3); + })); + profiler_->reportSamples(); + + auto& on_demand_logger = dynamic_cast(*onDemandLoggers_[0]); + EXPECT_CALL(on_demand_logger, handleSample(0, _, _)).Times(1); + profiler_->reportOnDemandSamples(); + + EXPECT_CALL(*cuptiEvents_, disableGroupSet(_)).Times(1); +} diff --git a/tb_plugins/profiling/libkineto/test/LoggerObserverTest.cpp b/tb_plugins/profiling/libkineto/test/LoggerObserverTest.cpp new file mode 100644 index 000000000..30ba4a824 --- /dev/null +++ b/tb_plugins/profiling/libkineto/test/LoggerObserverTest.cpp @@ -0,0 +1,96 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#include +#include + +// TODO(T90238193) +// @lint-ignore-every CLANGTIDY facebook-hte-RelativeInclude +#include "include/libkineto.h" +#include "src/Logger.h" +#include "LoggerCollector.h" + +using namespace KINETO_NAMESPACE; + +#if !USE_GOOGLE_LOG + +constexpr char InfoTestStr[] = "Checking LOG(INFO)"; +constexpr char WarningTestStr[] = "Checking LOG(WARNING)"; +constexpr char ErrorTestStr[] = "Checking LOG(ERROR)"; + +TEST(LoggerObserverTest, SingleCollectorObserver) { + // Add a LoggerObserverCollector to collect all logs during the trace. + std::unique_ptr lCollector = std::make_unique(); + Logger::addLoggerObserver(lCollector.get()); + + LOG(INFO) << InfoTestStr; + LOG(WARNING) << WarningTestStr; + LOG(ERROR) << ErrorTestStr; + + auto LoggerMD = lCollector->extractCollectorMetadata(); + EXPECT_TRUE(LoggerMD[LoggerOutputType::INFO][0].find(InfoTestStr) != std::string::npos); + EXPECT_TRUE(LoggerMD[LoggerOutputType::WARNING][0].find(WarningTestStr) != std::string::npos); + EXPECT_TRUE(LoggerMD[LoggerOutputType::ERROR][0].find(ErrorTestStr) != std::string::npos); + + Logger::removeLoggerObserver(lCollector.get()); +} + +#define NUM_OF_MESSAGES_FOR_EACH_TYPE 10 +#define NUM_OF_WRITE_THREADS 200 + +// Writes NUM_OF_MESSAGES_FOR_EACH_TYPE messages for each INFO, WARNING, and ERROR. +// NOLINTNEXTLINE(clang-diagnostic-unused-parameter) +void* writeSeveralMessages(void* ptr) { + for(int i=0; i lc1 = std::make_unique(); + std::unique_ptr lc2 = std::make_unique(); + std::unique_ptr lc3 = std::make_unique(); + std::unique_ptr lc4 = std::make_unique(); + Logger::addLoggerObserver(lc1.get()); + Logger::addLoggerObserver(lc2.get()); + Logger::addLoggerObserver(lc3.get()); + Logger::addLoggerObserver(lc4.get()); + + // Launch NUM_OF_WRITE_THREADS threads writing several messages. + pthread_t ListOfThreads[NUM_OF_WRITE_THREADS]; + for (int i=0; iextractCollectorMetadata(); + int InfoCount = 0, WarnCount = 0, ErrorCount = 0; + for (auto& md : lc1MD) { + InfoCount += md.first == LoggerOutputType::INFO ? md.second.size() : 0; + WarnCount += md.first == LoggerOutputType::WARNING ? md.second.size() : 0; + ErrorCount += md.first == LoggerOutputType::ERROR ? md.second.size() : 0; + } + + EXPECT_EQ(InfoCount, NUM_OF_WRITE_THREADS * NUM_OF_MESSAGES_FOR_EACH_TYPE); + EXPECT_EQ(WarnCount, NUM_OF_WRITE_THREADS * NUM_OF_MESSAGES_FOR_EACH_TYPE); + EXPECT_EQ(ErrorCount, NUM_OF_WRITE_THREADS * NUM_OF_MESSAGES_FOR_EACH_TYPE); + + Logger::removeLoggerObserver(lc1.get()); + Logger::removeLoggerObserver(lc2.get()); + Logger::removeLoggerObserver(lc3.get()); + Logger::removeLoggerObserver(lc4.get()); +} + +#endif // !USE_GOOGLE_LOG + +int main(int argc, char **argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/tb_plugins/profiling/libkineto/test/MockActivitySubProfiler.cpp b/tb_plugins/profiling/libkineto/test/MockActivitySubProfiler.cpp new file mode 100644 index 000000000..89f1d536c --- /dev/null +++ b/tb_plugins/profiling/libkineto/test/MockActivitySubProfiler.cpp @@ -0,0 +1,49 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#include +#include +#include + +#include "test/MockActivitySubProfiler.h" + +namespace libkineto { + +const std::set supported_activities {ActivityType::CPU_OP}; +const std::string profile_name{"MockProfiler"}; + +void MockProfilerSession::processTrace(ActivityLogger& logger) { + for (const auto& activity: activities()) { + activity.log(logger); + } +} + +const std::string& MockActivityProfiler::name() const { + return profile_name; +} + +const std::set& MockActivityProfiler::availableActivities() const { + return supported_activities; +} + +MockActivityProfiler::MockActivityProfiler( + std::vector& activities) : + test_activities_(activities) {}; + +std::unique_ptr MockActivityProfiler::configure( + const std::set& /*activity_types*/, + const Config& /*config*/) { + auto session = std::make_unique(); + session->set_test_activities(std::move(test_activities_)); + return session; +}; + +std::unique_ptr MockActivityProfiler::configure( + int64_t /*ts_ms*/, + int64_t /*duration_ms*/, + const std::set& activity_types, + const Config& config) { + return configure(activity_types, config); +}; + +} // namespace libkineto + diff --git a/tb_plugins/profiling/libkineto/test/MockActivitySubProfiler.h b/tb_plugins/profiling/libkineto/test/MockActivitySubProfiler.h new file mode 100644 index 000000000..36eaa13d1 --- /dev/null +++ b/tb_plugins/profiling/libkineto/test/MockActivitySubProfiler.h @@ -0,0 +1,72 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#pragma once + +#include +#include +#include + +#include "include/IActivityProfiler.h" + +namespace libkineto { + +class MockProfilerSession: public IActivityProfilerSession { + + public: + explicit MockProfilerSession() {} + + void start() override { + start_count++; + status_ = TraceStatus::RECORDING; + } + + void stop() override { + stop_count++; + status_ = TraceStatus::PROCESSING; + } + + std::vector& activities() override { + return test_activities_; + } + + std::vector errors() override { + return {}; + } + + void processTrace(ActivityLogger& logger) override; + + void set_test_activities(std::vector&& acs) { + test_activities_ = std::move(acs); + } + + int start_count = 0; + int stop_count = 0; + private: + std::vector test_activities_; +}; + + +class MockActivityProfiler: public IActivityProfiler { + + public: + explicit MockActivityProfiler(std::vector& activities); + + const std::string& name() const override; + + const std::set& availableActivities() const override; + + std::unique_ptr configure( + const std::set& activity_types, + const Config& config) override; + + std::unique_ptr configure( + int64_t ts_ms, + int64_t duration_ms, + const std::set& activity_types, + const Config& config) override; + + private: + std::vector test_activities_; +}; + +} // namespace libkineto diff --git a/tb_plugins/profiling/libkineto/test/PidInfoTest.cpp b/tb_plugins/profiling/libkineto/test/PidInfoTest.cpp new file mode 100644 index 000000000..b86cfb36d --- /dev/null +++ b/tb_plugins/profiling/libkineto/test/PidInfoTest.cpp @@ -0,0 +1,27 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#include "include/ThreadUtil.h" + +#include +#include + +#include +#include + +using namespace KINETO_NAMESPACE; + +TEST(ThreadNameTest, setAndGet) { + setThreadName("ThreadNameTest"); + EXPECT_EQ(getThreadName(), "ThreadNameTest"); + + setThreadName(""); + EXPECT_EQ(getThreadName(), ""); + + // Spaces etc are ok + setThreadName("Name w/ spaces"); + EXPECT_EQ(getThreadName(), "Name w/ spaces"); + + // More than 16 chars is not OK + setThreadName("More than 16 characters"); + EXPECT_EQ(getThreadName(), "Name w/ spaces"); +} diff --git a/tb_plugins/profiling/tb_plugin/.flake8 b/tb_plugins/profiling/tb_plugin/.flake8 new file mode 100644 index 000000000..1c5254b9f --- /dev/null +++ b/tb_plugins/profiling/tb_plugin/.flake8 @@ -0,0 +1,3 @@ +[flake8] +max-line-length = 120 +per-file-ignores = __init__.py:F401 torch_tb_profiler/io/file.py: F401 diff --git a/tb_plugins/profiling/tb_plugin/.gitignore b/tb_plugins/profiling/tb_plugin/.gitignore new file mode 100644 index 000000000..dc7d4e627 --- /dev/null +++ b/tb_plugins/profiling/tb_plugin/.gitignore @@ -0,0 +1,4 @@ +/build +/dist +/*.egg-info +__pycache__ diff --git a/tb_plugins/profiling/tb_plugin/.pre-commit-config.yaml b/tb_plugins/profiling/tb_plugin/.pre-commit-config.yaml new file mode 100644 index 000000000..a650ec832 --- /dev/null +++ b/tb_plugins/profiling/tb_plugin/.pre-commit-config.yaml @@ -0,0 +1,34 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# ------------------------------------------------------------------------- +default_language_version: + python: python3.8 + +ci: + autofix_prs: true + autoupdate_commit_msg: '[pre-commit.ci] pre-commit suggestions' + autoupdate_schedule: quarterly + # submodules: true + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.1.0 + hooks: + - id: end-of-file-fixer + exclude: torch_tb_profiler/static/index.html + - id: trailing-whitespace + - id: double-quote-string-fixer + + - repo: https://github.com/pre-commit/mirrors-autopep8 + rev: v1.6.0 + hooks: + - id: autopep8 + name: Format code + - repo: https://github.com/PyCQA/flake8 + rev: 4.0.1 + hooks: + - id: flake8 + args: + - "--max-line-length=120" + - "--per-file-ignores=__init__.py:F401 tb_plugin/torch_tb_profiler/io/file.py: F401" + name: Check PEP8 diff --git a/tb_plugins/profiling/tb_plugin/LICENSE b/tb_plugins/profiling/tb_plugin/LICENSE new file mode 100644 index 000000000..edb179715 --- /dev/null +++ b/tb_plugins/profiling/tb_plugin/LICENSE @@ -0,0 +1,33 @@ +BSD License + +For Kineto software + +Copyright (c) Facebook, Inc. and its affiliates. All rights reserved. + +All contributions by Microsoft: +Copyright (c) Microsoft Corporation. (The Azure AI Platform team) + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Facebook nor the names of its contributors may be used to + endorse or promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/tb_plugins/profiling/tb_plugin/README.md b/tb_plugins/profiling/tb_plugin/README.md new file mode 100644 index 000000000..e3c00875d --- /dev/null +++ b/tb_plugins/profiling/tb_plugin/README.md @@ -0,0 +1,478 @@ +# PyTorch Profiler TensorBoard Plugin + +This is a TensorBoard Plugin that provides visualization of PyTorch profiling. +It can parse, process and visualize the PyTorch Profiler's dumped profiling result, +and give optimization recommendations. + +### Quick Installation Instructions + +* Install from pypi + + `pip install torch-tb-profiler` + +* Or you can install from source + + Clone the git repository: + + `git clone https://github.com/pytorch/kineto.git` + + Navigate to the `kineto/tb_plugin` directory. + + Install with command: + + `pip install .` + +* Build the wheel + - `python setup.py build_fe sdist bdist_wheel` \ + **_Note_**: the build_fe step need setup yarn and Node.js + - `python setup.py sdist bdist_wheel` + +### Quick Start Instructions + +* Prepare profiling data + + We have prepared some sample profiling data at [kineto/tb_plugin/samples](./samples) + You can download it directly. + Or you can generate these profiling samples yourself by running + [kineto/tb_plugin/examples/resnet50_profiler_api.py](./examples/resnet50_profiler_api.py). + Also you can learn how to profile your model and generate profiling data from [PyTorch Profiler](https://pytorch.org/tutorials/intermediate/tensorboard_profiler_tutorial.html?highlight=tensorboard). + + Note: The recommended way to produce profiling data is assigning `torch.profiler.tensorboard_trace_handler` + to `on_trace_ready` on creation of `torch.profiler.profile`. + +* Start TensorBoard + + Specify the profiling data folder to `logdir` in TensorBoard. If you use the above samples data, start TensorBoard with: + + `tensorboard --logdir=./samples` + + If your web browser is not in the same machine that you start TensorBoard, + you can add `--bind_all` option, such as: + + `tensorboard --logdir=./samples --bind_all` + + Note: Make sure the default port 6006 is open to the browser's host. + +* Open TensorBoard in Chrome browser + + Open URL `http://localhost:6006` in the browser. + If you use `--bind_all` in tensorboard start command, the hostname may not be 'localhost'. You may find it in the log printed after the cmd. + +* Navigate to the PYTORCH_PROFILER tab + + If the files under `--logdir` are too big or too many, + please wait a while and refresh the browser to check latest loaded result. + +* Loading profiling data from the cloud + * AWS S3 (S3://) + + Install `boto3`. Set environment variables: `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`. Optionally, `S3_ENDPOINT` can be set as well.\ + For minio, the S3 url should start with the bucket name `s3:////` instead of minio prefix `s3://minio//`. At the same time, the `S3_ENDPOINT` is needed as well. \ + Follow these guides to get set-up with minio: + * Server: https://docs.min.io/docs/minio-quickstart-guide.html + * MC Client: https://docs.min.io/docs/minio-client-quickstart-guide.html + + For example, the following commands can be used to create minio storage: + ```bash + ./mc alias set s3 http://10.150.148.189:9000 minioadmin minioadmin + ./mc mb s3/profiler --region=us-east-1 + ./mc cp ~/notebook/version_2 s3/profiler/ --recursive + export AWS_ACCESS_KEY_ID=minioadmin + export AWS_SECRET_ACCESS_KEY=minioadmin + export AWS_REGION=us-east-1 + export S3_USE_HTTPS=0 + export S3_VERIFY_SSL=0 + export S3_ENDPOINT=http://localhost:9000 + tensorboard --logdir=s3://profiler/version_2/ --bind_all + ``` + + * Azure Blob (https://\.blob.core.windows.net) + + Install `azure-storage-blob`. Optionally, set environment variable `AZURE_STORAGE_CONNECTION_STRING`. + + * Google Cloud (GS://) + + Install `google-cloud-storage`. + + --- + > **_NOTES:_** For AWS S3, Google Cloud and Azure Blob, the trace files need to be put on a top level folder under bucket/container. + --- + + We prepared some sample data in blob, you can also access it using the command + + tensorboard --logdir=https://torchtbprofiler.blob.core.windows.net/torchtbprofiler/demo/ --bind_all + + and open TensorBoard your browser to see all the views described below. + + Note: for accessing data in Azure Blob, you need to install torch-tb-profiler with `pip install torch-tb-profiler[blob]` + +### Quick Usage Instructions + +We regard each running with profiler enabled as a "run". +In most cases a run is a single process. If DDP is enabled, then a run includes multiple processes. +We name each process a "worker". + +Each run corresponds to a sub-folder under the folder specified by "--logdir". +Each sub-folder contains one or more chrome trace files, one for each process. +The kineto/tb_plugin/samples is an example of how the files are organized. + +You can select the run and worker on the left control panel. + +![Alt text](./docs/images/control_panel.PNG) + +Runs: Select a run. Each run is one execution of a PyTorch application with profiling enabled. + +Views: We organize the profiling result into multiple views, +from coarse-grained (overview-level) to fine-grained (kernel-level). + +Workers: Select a worker. Each worker is a process. There could be multiple workers when DDP is used. + +Span: There may be multiple profiling trace files of different spans to be generated when using [torch.profiler.schedule](https://github.com/pytorch/pytorch/blob/master/torch/profiler/profiler.py#L24) as schedule of torch.profiler. +You can select them with this selection box. + +Currently we have the following performance diagnosis views: +- Overall View +- Operator View +- Kernel View +- Trace View +- Memory View +- Distributed View + +We describe each of these views below. + +* Overall View + + The overall view is a top level view of the process in your profiling run. + It shows an overview of time cost, including both host and GPU devices. + You can select the current worker in the left panel's "Workers" dropdown menu. + + An example of overall view: + ![Alt text](./docs/images/overall_view.PNG) + + The 'GPU Summary' panel shows GPU information and usage metrics of this run, include name, global memory, compute capability of this GPU. + The 'GPU Utilization', 'Est. SM Efficiency' and 'Est. Achieved Occupancy' shows GPU usage efficiency of this run at different levels. + The 'Kernel Time using Tensor Cores' shows percent of the time Tensor Core kernels are active. + The detailed information about the above four metrics can be found at [gpu_utilization](./docs/gpu_utilization.md). + + The 'Step Time Breakdown' panel shows the performance summary. We regard each iteration (usually a mini-batch) as a step. + The time spent on each step is broken down into multiple categories as follows: + + 1. Kernel: Kernels execution time on GPU device; + + 2. Memcpy: GPU involved memory copy time (either D2D, D2H or H2D); + + 3. Memset: GPU involved memory set time; + + 4. Communication: Communication time only appear in DDP case; + + 5. Runtime: CUDA runtime execution time on host side; + Such as cudaLaunchKernel, cudaMemcpyAsync, cudaStreamSynchronize, ... + + 6. DataLoader: The data loading time spent in PyTorch DataLoader object; + + 7. CPU Exec: Host compute time, including every PyTorch operator running time; + + 8. Other: The time not included in any of the above. + + Note: The summary of all the above categories is end-to-end wall-clock time. + + The above list is ranked by priority from high to low. We count time in priority order. + The time cost with highest priority category(Kernel) is counted first, + then Memcpy, then Memset, ..., and Other is counted last. + In the following example, the "Kernel" is counted first as 7-2=5 seconds; + Then the "Memcpy" is counted as 0 seconds, because it is fully hidden by "Kernel"; + Then "CPU Exec" is counted as 2-1=1 seconds, because the [2,3] interval is hidden by "Kernel", only [1,2] interval is counted. + + In this way, summarization of all the 7 categories' counted time in a step + will be the same with this step's total wall clock time. + + ![Alt text](./docs/images/time_breakdown_priority.PNG) + + Performance Recommendation: Leverage the profiling result to automatically highlight likely bottlenecks, + and give users actionable optimization suggestions. + +* Operator View + + This view displays the performance of every PyTorch operator that is executed either on the host or device. + + ![Alt text](./docs/images/operator_view.PNG) + Each table row is a PyTorch operator, which is a computation operator implemented by C++, + such as "aten::relu_", "aten::convolution". + + Calls: How many times the operator is called in this run. + + Device Self Duration: The accumulated time spent on GPU, not including this operator’s child operators. + + Device Total Duration: The accumulated time spent on GPU, including this operator’s child operators. + + Host Self Duration: The accumulated time spent on Host, not including this operator’s child operators. + + Host Total Duration: The accumulated time spent on Host, including this operator’s child operators. + + Tensor Cores Eligible: Whether this operator is eligible to use Tensor Cores. + + Tensor Cores Self (%): Time of self-kernels with Tensor Cores / Time of self-kernels. + Self-kernels don't include kernels launched by this operator’s child operators. + + Tensor Cores Total (%): Time of kernels with Tensor Cores / Time of kernels. + + CallStack: All call stacks of this operator if it has been recorded in profiling trace file. + To dump this call stack information, you should set the 'with_stack' parameter in torch.profiler API. + The TensorBoard has integrated to VSCode, if you launch TensorBoard in VSCode, clicking this CallStack will forward to corresponding line of source code as below: + + ![Alt text](./docs/images/vscode_stack.PNG) + + Note: Each above duration means wall-clock time. It doesn't mean the GPU or CPU during this period is fully utilized. + + The top 4 pie charts are visualizations of the above 4 columns of durations. + They make the breakdowns visible at a glance. + Only the top N operators sorted by duration (configurable in the text box) will be shown in the pie charts. + + The search box enables searching operators by name. + + "Group By" could choose between "Operator" and "Operator + Input Shape". + The "Input Shape" is shapes of tensors in this operator’s input argument list. + The empty "[]" means argument with scalar type. + For example, "[[32, 256, 14, 14], [1024, 256, 1, 1], [], [], [], [], [], [], []]" + means this operator has 9 input arguments, + 1st is a tensor of size 32\*256\*14\*14, + 2nd is a tensor of size 1024\*256\*1\*1, + the following 7 ones are scalar variables. + + ![Alt text](./docs/images/operator_view_group_by_inputshape.PNG) + +* Kernel View + + This view shows all kernels’ time spent on GPU. + The time is calculated by subtracting the kernel's start time from the end time. + + Note: This view does not include cudaMemcpy or cudaMemset. Because they are not kernels. + + ![Alt text](./docs/images/kernel_view.PNG) + + * Tensor Cores Used: Whether this kernel uses Tensor Cores. + + * Total Duration: The accumulated time of all calls of this kernel. + + * Mean Duration: The average time duration of all calls. That's "Total Duration" divided by "Calls". + + * Max Duration: The maximum time duration among all calls. + + * Min Duration: The minimum time duration among all calls. + + Note: These durations only include a kernel's elapsed time on GPU device. + It does not mean the GPU is fully busy executing instructions during this time interval. + Some of the GPU cores may be idle due to reasons such as memory access latency or insufficient parallelism. + For example, there may be insufficient number of available warps per SM for the GPU to effectively + hide memory access latencies, or some SMs may be entirely idle due to an insufficient number of blocks. + Please refer to [Nvidia's best-practices guide](https://docs.nvidia.com/cuda/cuda-c-best-practices-guide/index.html). + To investigate efficiency for each kernel, we calculate and show the 'Mean Blocks Per SM' and 'Mean Est. Achieved Occupancy' in the last two column. + + * Mean Blocks Per SM: Blocks per SM = Blocks of this kernel / SM number of this GPU. If this number is less than 1, it indicates the GPU multiprocessors are not fully utilized. "Mean Blocks per SM" is weighted average of all runs of this kernel name, using each run’s duration as weight. + + * Mean Est. Achieved Occupancy: The definition of Est. Achieved Occupancy can refer to [gpu_utilization](./docs/gpu_utilization.md), It is weighted average of all runs of this kernel name, using each run’s duration as weight. + + The top left pie chart is a visualization of "Total Duration" column. + It makes the breakdowns visible at a glance. + Only the top N kernels sorted by accumulated time (configurable in the text box) will be shown in the pie chart. + + The top right pie chart is percent of the kernel time using and without using Tensor Cores. + + The search box enables searching kernels by name. + + "Group By" could choose between "Kernel Name" and "Kernel Properties + Op Name". + + "Kernel Name" will group kernels by kernel name. + + "Kernel Properties + Op Name" will group kernels by combination of kernel name, launching operator name, + grid, block, registers per thread, and shared memory. + + ![Alt text](./docs/images/trace_view.PNG) + + * Operator: The name of PyTorch operator which launches this kernel. + + * Grid: Grid size of this kernel. + + * Block: Block size of this kernel. + + * Register Per Thread: Number of registers required for each thread executing the kernel. + + * Shared Memory: Sum of dynamic shared memory reserved, and static shared memory allocated for this kernel. + +* Trace View + + This view shows timeline using the chrome tracing plugin. Each horizontal area represents a thread or a CUDA stream. + Each colored rectangle represents an operator, or a CUDA runtime, or a GPU op which executes on GPU + (such as a kernel, a CUDA memory copy, a CUDA memory set, ...) + + ![Alt text](./docs/images/trace_view.PNG) + + In the above example: + + The "thread 25772" is the CPU thread that do "backward" of neural network. + + The "thread 25738" is the main CPU thread, which mainly do data loading, forward of neural network, and model update. + + The "stream 7" is a CUDA stream, which shows all kernels of this stream. + + You can see there are 6 "ProfilerStep" at the top of "thread 1". Each "ProfilerStep" represents a mini-batch step. + + The suspended toolbar has functionalities to help view the trace line. + For example, when the up-down arrow is enabled, + you can zoom in by dragging the mouse up and keeping mouse's left button pushed down. + + ![Alt text](./docs/images/trace_view_one_step.PNG) + + The "Optimizer.step#SGD.step" and "enumerate(DataLoader)#_SingleProcessDataLoaderIter.\__next\__" + are high-level python side functions. + + When you select the top-right corner's "Flow events" to "async", + you can see the relationship between an operator and its launched kernels. + ![Alt text](./docs/images/trace_view_launch.PNG) + + You can also view the gpu utilization and Est. SM Efficiency in the trace view. They are drawn alongside the timeline: + + ![Alt text](./docs/images/trace_view_gpu_utilization.PNG) + + When you select the top-right corner's "Flow events" to "fwd_bwd_correlation", + you can see the relationship between forward operator and its launched backward operator. + Note: Only the backward operator's direct launching forward operator will be connected by line, + its ancestor operators which call this operator as child will not be connected. + ![Alt text](./docs/images/trace_view_fwd_bwd_correlation.PNG) + +* Memory View + + The Pytorch profiler records all memory allocation/release events and allocator's internal state during profiling. For + each operator, the plugin aggregates all the events inside its lifespan. + + ![Alt text](./docs/images/memory_view.PNG) + + The memory kind could be selected in 'Device' selection box. For example, 'GPU0' means the following plot and tables only shows each + operator's memory usage on GPU 0, not including CPU or other GPUs. + + * Memory Curve + + Memory curve shows the memory usage trends. It helps the user get an overview on memory consumption. The 'Allocated' plot is the + total memory requested from the allocator, for example, used by tensors. The 'Reserved' plot only makes sense if the underlying + allocator make use of caching mechanism. It represents the total memory that is allocated from the operating system by the allocator. + + User can select on the memory curve plot and zoom into the selected range by pressing left mouse button and dragging on the curve. + Right click will reset the plot to the initial state. The selection will affect 'Memory Events' table and 'Memory Statistics' table + as mentioned in the following sections. + + * Memory Events + + Memory events table shows the memory allocation and release event pairs. Definition of each field in the table: + + * Operator: The immediate operator causing allocation from allocator. In pytorch, some operators such as + `aten::empty` is widely used as an API for tensor creation, in this case, we show it as ` ()`. + + * Size: The allocated memory size. + + * Allocation Time: Memory allocation time point relative to profiler start. It maybe missing from the table if the allocation event + is not included in the selected range. + + * Release Time: Memory deallocation time point relative to profiler start. It maybe missing from the table if the release event is + not included in the selected range. Notice, released memory block might still be cached by the underlying allocator. + + * Duration: The life duration of the allocated memory. It maybe missing from the table if Allocation Time or Release Time is absent. + + * Memory Statistics + + Definition of each field in the table: + + * Calls: How many times this operator is called. + + * Size Increase: The memory increase size includes all children operators. It sums up all allocation bytes and minus all the memory release bytes. + + * Self Size Increase: The memory increase size associated with the operator itself excluding that of its children. It sums up all allocation bytes and minus all the memory release bytes. + + * Allocation Count: The allocation count including all children operators. + + * Self Allocation Count: The allocation count belonging to the operator itself excluding its children. + + * Allocation Size: The allocation size including all children operators. It sums up all allocation bytes without considering the memory free. + + * Self Allocation Size: The allocation size belonging to the operator itself. It sums up all allocation bytes without considering the memory free. + + +* Distributed View + + This view will appear automatically only for DDP jobs that use nccl for communication. + There are four panels in this view: + + ![Alt text](./docs/images/distributed_view.PNG) + + * The top panel shows the information about nodes/processes/GPU hierarchy of this job. + + * The left panel in the middle is 'Computation/Communication Overview'. Definition of each legend: + * Computation: the sum of kernel time on GPU minus the overlapping time. + * Overlapping: the overlapping time of computation and communication. More overlapping represents better parallelism between computation and communication. Ideally the communication would be totally overlapped with computation. + * Communication: the total communication time minus the overlapping time. + * Other: step time minus computation and communication time. Maybe includes initialization, data loader, CPU computation, and so on. + + From this view, you can know computation-to-communication ratio of each worker and load balance between workers. For example, if the computation + overlapping time of +one worker is much larger than others, there may be a problem of loading balance or this worker may be a straggler. + + * The right panel in the middle is 'Synchronizing/Communication Overview'. Definition of each legend: + * Data Transfer Time: part in the total communication time for actual data exchanging. + * Synchronizing Time: part in the total communication time for waiting and synchronizing with other workers. + + From this view, you can know the efficiency of communication (how much ratio of total communication time is really used for exchanging data and how much is just waiting for data from other workers) + + * The 'Communication Operations Stats' summarizes the detailed statistics of all communication ops in each worker. Definition of each field: + * Calls: How many times this operator is called in this run. + * Total Size (bytes): Total data size transferred in operators of this type. + * Avg Size (bytes): Average data size transferred in each operator of this type. + * Total Latency (us): Total latency of all operators of this type. + * Avg Latency (us): Average latency of each operator of this type. + * Data Transfer Time (us): Total time actually used for data transfer in operator of this type. + * Ave Data Transfer Time (us): Average time actually used for data transfer in each operator of this type. + +* Module View + + If the torch.nn.Module information is dumped into the result Chrome tracing file by Pytorch profiler, the plugin could display the nn.Module hierarchy and summary. + + ![Alt text](./docs/images/module_view.png) + + * The top table shows each torch.nn.Module statistics information including: + * Occurrences: how many times the module is called in the training process. + * Operators: how many operators the module invokes. + * Host Total Time: The accumulated time spent on Host, including the child submodule. + * Host Self Time: The accumulated time spent on Host, not including the child submodule. + * Device Total Time: The accumulated time spent on GPU of the operators contained in the module, including the child submodule. + * Device Self Time: The accumulated time spent on GPU of the operators contained in the module, not including the child submodule. + + * The middle flamegraph shows the torch.nn.Module hierarchy information + * The bottom graph shows the main thread operators tree. + +* Lightning View + + If the Chrome tracing file is from PytorchLightning job, the plugin will show a Lightning View which is customized for Pytorch Lightning. + All the data of this view is from PytorchLightning framework. + + ![Alt text](./docs/images/lightning_view.png) + + * The top table shows the model structure. The meaning of metrics in the table is same as Module View. + * The middle flamegraph shows the model hierarchy information. + * The bottom graph shows the call tree of all hooks in PytorchLightning. + +* Diff Run View + + The diff run feature helps to compare two run by logical timeline. The key comparision operators include backward, dataloader, torch.nn.Module, optimizer. If each operator contains these sub-operators internally, the diff run could be zoom in by click the bar. + + ![Alt text](./docs/images/diff_view.png) + + * The top bar chart shows each operator type and trend comparision result. + * The middle line chart shows the delta and accumulated execution time difference against each operator type. + * The bottom table show the operators difference for the following categories: + * Host Total Duration: The accumulated time spent on Host, including this operator’s child operators. + * Host Self Duration: The accumulated time spent on Host, not including this operator’s child operators. + * Device Total Duration: The accumulated time spent on GPU, including this operator’s child operators. + * Device Self Duration: The accumulated time spent on GPU, not including this operator’s child operators. + +### PyTorch Profiler TensorBoard Plugin 0.2 Release Notes + +Known Issues: This software does not support Python 3.9.0, 3.9.1, 3.9.2. +If the TensorBoard launching reports error message "ImportError" and "circular import", +please update your Python to higher version. diff --git a/tb_plugins/profiling/tb_plugin/ci_scripts/install_env.sh b/tb_plugins/profiling/tb_plugin/ci_scripts/install_env.sh new file mode 100644 index 000000000..11f588a03 --- /dev/null +++ b/tb_plugins/profiling/tb_plugin/ci_scripts/install_env.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +set -ex + +# install cuda +#if [ "$CUDA_VERSION" = "cu101" ]; then +# wget https://developer.download.nvidia.com/compute/cuda/10.1/Prod/local_installers/cuda_10.1.243_418.87.00_linux.run +# sudo sh cuda_10.1.243_418.87.00_linux.run +#elif [ "$CUDA_VERSION" = "cu102" ]; then +# wget https://developer.download.nvidia.com/compute/cuda/10.2/Prod/local_installers/cuda_10.2.89_440.33.01_linux.run +# sudo sh cuda_10.2.89_440.33.01_linux.run +#elif [ "$CUDA_VERSION" = "cu111" ]; then +# wget https://developer.download.nvidia.com/compute/cuda/11.1.0/local_installers/cuda_11.1.0_455.23.05_linux.run +# sudo sh cuda_11.1.0_455.23.05_linux.run +#elif [ "$CUDA_VERSION" = "cu112" ]; then +# wget https://developer.download.nvidia.com/compute/cuda/11.2.0/local_installers/cuda_11.2.0_460.27.04_linux.run +# sudo sh cuda_11.2.0_460.27.04_linux.run +#fi + + + +# install pytorch +pip install numpy tensorboard typing-extensions pillow pytest +if [ "$PYTORCH_VERSION" = "nightly" ]; then + pip install --pre torch -f "https://download.pytorch.org/whl/nightly/$CUDA_VERSION/torch_nightly.html" + pip install --pre torchvision --no-deps -f "https://download.pytorch.org/whl/nightly/$CUDA_VERSION/torch_nightly.html" +elif [ "$PYTORCH_VERSION" = "1.11rc" ]; then + pip install --pre torch -f "https://download.pytorch.org/whl/test/$CUDA_VERSION/torch_test.html" + pip install --pre torchvision --no-deps -f "https://download.pytorch.org/whl/test/$CUDA_VERSION/torch_test.html" +elif [ "$PYTORCH_VERSION" = "stable" ]; then + pip install torch torchvision +fi + +python -c "import torch; print(torch.__version__, torch.version.git_version); from torch.autograd import kineto_available; print(kineto_available())" diff --git a/tb_plugins/profiling/tb_plugin/docs/gpu_utilization.md b/tb_plugins/profiling/tb_plugin/docs/gpu_utilization.md new file mode 100644 index 000000000..c4f45b880 --- /dev/null +++ b/tb_plugins/profiling/tb_plugin/docs/gpu_utilization.md @@ -0,0 +1,22 @@ +* GPU Utilization: GPU busy time / all steps time. The higher, the better. All steps time is the total time of all profiler steps(or called as iterations). + GPU busy time is the time during “all steps time” when is at least one GPU kernel running on this GPU. + However, this high-level utilization metric is coarse. It can’t tell how many SMs(Stream Multiprocessors) are in use. + For example, a kernel with a single thread running continuously will get 100% GPU utilization. + +* Est. SM Efficiency: Estimated Stream Multiprocessor Efficiency. The higher, the better. This metric of a kernel, SM_Eff_K = min(blocks of this kernel / SM number of this GPU, 100%). + This overall number is the sum of all kernels' SM_Eff_K weighted by kernel's execution duration, divided by “all steps time”. + It shows GPU Stream Multiprocessors’ utilization. + Although it is finer grained than above “GPU Utilization”, it still can’t tell the whole story. + For example, a kernel with only one thread per block can’t fully utilize each SM. + +* Est. Achieved Occupancy: For most cases such as memory bandwidth bound kernels, a higher value often translates to better performance, especially when the initial value is very low. [Reference](http://developer.download.nvidia.com/GTC/PDF/GTC2012/PresentationPDF/S0514-GTC2012-GPU-Performance-Analysis.pdf). The definition of occupancy is [here](https://docs.nvidia.com/gameworks/content/developertools/desktop/analysis/report/cudaexperiments/kernellevel/achievedoccupancy.htm). + Occupancy is the ratio of active warps on an SM to the maximum number of + active warps supported by the SM. The theoretical occupancy of a kernel is upper limit occupancy of this kernel, limited by multiple + factors such as kernel shape, kernel used resource, and the GPU compute capability. + Est. Achieved Occupancy of a kernel, OCC_K = min(threads of the kernel / SM number / max threads per SM, theoretical occupancy of the kernel). + This overall number is the weighted sum of all kernels OCC_K using kernel's execution duration as weight. It shows fine-grained low-level GPU utilization. + + * Kernel Time using Tensor Cores: Total GPU Time for Tensor Core kernels / Total GPU Time for all kernels. Higher is better. + Tensor Cores are mixed precision floating point operations available for Volta GPUs (Titan V) and beyond. + The cuDNN and cuBLAS libraries contain several Tensor Cores enabled GPU kernels for most Convolution and GEMM operations. + This number shows Tensor Cores usage time ratio among all kernels on a GPU. diff --git a/tb_plugins/profiling/tb_plugin/docs/images/control_panel.PNG b/tb_plugins/profiling/tb_plugin/docs/images/control_panel.PNG new file mode 100644 index 0000000000000000000000000000000000000000..31bd12d9ce7c0d5efa17056ea870de5e835a5031 GIT binary patch literal 16255 zcmdseXIPWXx^DRR6cquLrXXMir79v(5Ks{jX`=Lwf`~u}H3)==2q;LCCLlyaigamd z5ZZ^7ND&ATk`O` zzQ7@gLjV9kz~J7UhX4Q%1OWJB;=q3Hf6hdvr*Qx9_&?OY4XEfnN#j=jbiZYM3jp|* z$iL&Vk6S+ozGveP00_4I{_u486uSZdIwA&lZkY!=E|Fng4zo~-uaak6Uh^(dhj&x| zx(K-nwrUCieu{HG92nms5gmKH%ElO2aiaMz8$8MluP8_!wjjBd3|##9B+cKm%fNgj zURXZpcDvD>wdxa`Ydix{F2sJeRcdX-DXOSsNOLR2X(H4yH)Ti1TQclLug$fBE0l}^UYX^AC30*?DTC#C*akQk~K_+J-xr4*rRcRVz7{YeD zBlLHWvosj_`aI;G;&JpIl~=p9($GDX)^Q_1-Z?x0YbP%g*I8mpV45O z69iHQ%IKNZmllA9S}IzCm|Dz2T|#a&8}c2Cc*?W4{`27WmfW7Z*nInduSWcS3>ItV zpV2M!(AeWa(^~Wo7{kB1f!S6K-eS6Kk`bko6wE39z8$Qh46EvgDNI>MMb8YVcU6G- zxT>QZUso7`wdASHle%r+#wMZGXvbG{Vs0ib+^yYN`Y`fkU-%2wWt*LOOH?pDSPi63 zu<&2mn5^F7INF5FpY@|H0!s(4meo}6A;N96IYuMMO?;-xUj5DhS-5M#ou2H(DzEI$ zY*q8LC=%qn4cifh8_nl-Oq;8N?%t&v#9$u!?*Ps*OnQuC_LrkzA;`hFU zt&B_rTe(YTv}6VDzJ2%6phG|_1CF&bWa`nOWe|%*@^~I&YcgqRuYx$-Yu*NC6gmqH zpBdkPMvGyh!?fTWl2fe`sq%+Ps`RJFJ&l2uV$w|=oX`r5ok*tt;8rYaR0Y1Br3hog zaipk>wZYW%acM%%Q)T`aaM{UPZ~zr8YLsO<^q~^HsgI*OmeYI zBUuT;N%_N#uh?3GzRI_EGhK&iI^FFLdn9EwT2#^WA$7-ZVBis=vmI#Xv2W}Sb?nlz z+K*CdxhUUV@8GGTN`a;7fft7s5^WcRb+j^UEI4ytlqiHjzY|%uyX(tuT@5p}rTOWvBclSNs4gth4L8PDaKnI` zWn#H><@}A6>t+xcub1)y%p)L$i;X904G%1(rS1>Iyk^#>cMF=>^yGwg)sbK=_!#?U z47x+Al^t#-_C2U`5#wWCgvT<7lLE=$Ar)d_6|@4Q|#?T|vT zrLRxKpjL5Tu@wg);?3J`70!|YVMD7rYd`L6`hT02j=}8iw7ai~2XO#k{Bz_&K>H^J z<utb0;_n_S-rqzkj=XUZX z4b0=~`}m-zi>t%e?E!yH+8t)vI~Od(Mw%DBO^16KzKEKhYSl1r$U%84dp)UN3Up+5 z_dUX3SD8ztuCQs|`UcC*-1qVhAQ$wGnm}u#>o(e0fVsW@E1Il&dc00=td6W&Q78Qh zs)TWSjP}{>>(2<&;lw}NuGHRLbU?B0HQ8_PCfUCL!8_s4anKY+?bU2m_HN(QK&G$E z`l?m7D?L8N&tryo#>v03X?0DNxp`lvSj=`MFAQhfR@=Kp3aE29t5LB5_Nf=3dCDwr z2R^&X&wPB(we7w#>-i~y5Jaa;@fv^I1mLS>U0uaw98N_b%PV_5aA^LAfT>dl(R_2) zzMcIg2)b?QU2#9%svZ`HU)RDbB6heDrD&4ob$k;D&qno;#P?o$pXk1CKu^Y zw|evH0S=?wh}cuiVaLV+xK*Byr~$k+>@6k21A zUnyMlVH-y%+-xdQy=#EeBwNG4PQT*BRPPY>jFdTBc64_{qpmWEv6sZpO!3yG*-ffvGloUHynDE7DjN|%9 z`#i7mp`Wb_q+0Ey{FfJu7IZ%yQTDwsm)Uuau) zoaM1#)KdRaY~e-zdRNKM)5z(kP4=T@kC!rLq@Q}b|CNovl@EbVnqAb z1^gxMc!l;EzkRXyOlbYOG*ah$^y+YFN% z%7#KalF5B2-QdbnoxSU*famtg*pNF@aWcoY-}AExPDyb-M(E}}OPXqNU~!gMzyjX3 zU*uI8&hu=HJHpD4T5-GCvs_$Yqwdh=O$uVOB_=z2KYI6RiJ&L5=tKHmLg;#vtiy|- zN<|?~Y^S(73JjC?KneN0pN^DCNAVYZ-rjKKZPw7#j^f*SQJFogo2}3JT6=Iw9xxHZ@&L&CK)9ka(I;wMqdF|7T%z zy_;5Z+e}1nq{KLbF?LQw&n&h44C!L*=fh%D6EhLFssoVN&le)~kdH+4PAInKe7?}_ ze3Z_;Y=0cM=t8Wj^=nP;EiM0-Z^4&)_A%0?fv^j>S=cFlA&R4hfSwuI#+x6~xOMf< zu;aDXag@fQ02@h^1U-fzH$BR~XDYV2hZ%T*DNpZx_ZE%4Bxkip&o3C{c5@EC9!!XH z(se$sHrJhAmt0!FsR}br58p9H`7v}99quATxP2js&=~b=+N~K>K9QdC5*ctBXo7o; z(-LGZ@YkT2p`6Am%D*ypuG!e8dg||e4md%6$n6?$()=?xdtwWYK7yI&y&d`4OTtq$ zhP_Hu!M|(TYy}6k4T7>Jwm>D>r%9F4ETpEO$`q*8)=))DmS2cGQ)Cmy>LXWq*s!_~ zQ{7Ix_th)IS8X*`5aF!$vwOt8NA@Q6hDGqyZ50mAb*yhs7^Z9w+bg3qu`02{RF~}n z;^uc>R8Uyc4+e)Y5`dcSX-rn^POGnDc@?Zfv72R0W0R)b4$09v zV+G$LlU8fjcWkwp@9djab5%H~;9j@Bm-p{W#^yzwV9ZttY)HJ1Pd~9acz;YEXkqm_ zwCAmjAGk)if61CWvg^7@?R-fDMaCkB*N#96)iBk*HyX?Lpw0`qgLy-m|~ zJ4Z%tfiMua%NOb=?oqt0wb$x2M{svCEEFJ)ryn&HH!&f@;_`WSh0{!$)+f$}H&FIE znA_0FjQp16A0WsN>;mFZjHrs5LRwBf>+Sr!-kzh|%KcUBEG47#ET?*zus6Ik31`s) zSeF=hUHH8j)an(RYg_R5Yi3NR+QX>&B!R(64V|5%Fw*+R)CB@Lr1JbAhPGZONLUF( z^cYvJe~57hcCCib#o#8~V7+rq?bA`&fh)F$8KxPOZr@@5AO4J=J}F!T6%T@jBt*Te z%v2`J=2-S2sv6o?LOE*TA5Hu>9vhJ8@+!16QyuJBj_X?O#0Qo9$e*ei z;Q^hdgY)dUdRr!#AiU1PRBm}ZzrzQ_G@lf>3{w?(9d}vLt_=qdI6o1mYshE~Nly1$ z7JP&7GB@ zjoszauGK>6`}I9i9Y>~El}#1tVjMXkDX~{&NaIog=Cw=W*j1UZ{8Lz-bmJm8cqi0P z;2@O23a@AaKJxE%45jbM4x%J2p1r{ncOL+c(Kph+;=9>ziWup(1nvxfx?R-AUI_tb z^hs?fxoCOl-i*!>kHHK$59UCZP;BzS<>wl;=-mF2hogjY9RB8d`A@awjfo+T!0nK{ z^SqT^tEF5YN!$~nObFa6YdNkRO$+h2Ki~+PmD4U0@7OC-BW*mkJIQ_*s^i1*MU&bY z9A9-u)5R)>-9{-6Kl7sVC0dawuM;zoenmiBAk%gHJwv&lS--{UlLcx|k74M%4nfzh zYatXH-zqt6C5OVC*4P?~PCDQ0*;Ffg>m6fIBhU`%DH?f(|Fjl}zCWjKl>ALTyn67^ z?7fu+=4=DTW$s0NaDWx)V{}!tFEVe4SRXtRqHMo6a#6?O0NxAzL}-h7#P*X0^n*&S zvb|lo^lV7PiFBFh3}Y*Mt4(QgXlLLH+Vu(`W)Awh4?R!N70B)jW(yK@l%UbL11Wh#36~ue7;i7yRm|?7*prNZBq}r@%JvlwG;fxFsjalcMEkLg z?R>i%Gy2_+oAf?jxQ9Wv#+xAw*ugcvphoT20rukXW}xlHA8H@7Lk0={cVeRaUAE&d zLbNe=U*)QJETF8po|emuznyp7#Em%PBS^Ph!D+7`1Tw73j<{+xnPjJhKM#yguuSx~ z`L(^|$1YfGh$|Wl^w8~s&pPcg1DPq7zs4@Xf*sV$(?OvPdjZxX>VH}Iz!gu<^;DEv zj_DsCGp?(kV%PJJ>hw}_@~Z|9@pUH#twpC;(|Qw%;ya>vC2enofAF+$;(Uko^$1%% z+qWQa+ec*O8s+hzwviOvagO@hygfyBjE*Ml`vgp1_BFAa!}j4X5>4@v4jZjst=x!W zCcP8wrRAJunry9D+qH7=1R~_8&tKhttapd*ey`}el>QW(aZ*=+QHw84tCQWZewJY` za;E)IM_%v^@!d@wmZ^ywTkWH!i+vDB(Q&0iU-B4sq0QTBZUDvVl-m7H>b`uaqbAof zJx{b(4jN2jW?;Iu_P+sHv35@9(ph}7oO(84^h2D)iPOB@z3i$+7yHH~p%+O@O%|_1 z%|=uu8WTZffdY`F@5>|uD&rOHZTU1iCJ>pgc|~v;d`Y)`vh~)AQr~jFJea3Me0Cza z&t2#OZYDmXJWYzUKBdCn|A6}EqxPkB&ip7oC8+6HM(Y0O`O_Y0OZqyLE0~lxEeA(4 zYfE3Yq!O?KJRv??W4@x@=?m0af~aVB_7)OMQ#~lbHc?M(NtM(v1!57Ebg;a|%ONCe zL-wO~=b)+G8f?R}Yjt;KcrorkPlDOC+@2(e=Pu?Awx2Gm zoSYuwf3~M|gd>))>ECghF?Y!uHJlBssLG*E3CVqt&sF=|8xD1|?nzNZCu|+#(~B(X zWilMMgEq8K{7!^vyP%4|qp#gW5luh+CJ)Jl`)V7CZr*Bq;#3C4suAU=AH7>FXhit;-Ln zfB9^Dro5>erdHIgPj(FYQhHi%?Fl$(;a4=uyolA#9;@Y24!MkM0+r?;5prvIKJ<{q zVN8g{N98hoN(vSzHG@g3Xs}xQp%2?68-qTYI@Qpo*~x4z&q($QC?D@7hb(LVc9o1apag2auswIawJW z726aj%qnP$DvgyhzSd&yt6Aqq%&I5(yDWjOQ9lbzt2n0R8lgB@`)?Ijd-Kb=uT0=2 z)5fZHkaG&|dFv2?9i*5FE8g10DSEBYD+}4W;#qIn>U%+xQRCx|T65I3wIg8DN*pB3 z;DICcTy#Z3NrbnJz?3YT-Y-Z1LVoP5_L`o<4emE!y~H{u)v8EQ-c$Kb-;IB~zk-);uyzuD%89Ka``S$m6HCIiV zD&)TDyqO7O#-c9p`#@$oK}AUsb9XmaD+p-eQlH$RUC* zZcD!9`mdxO%i=~OvaN%8Nil4XGmi*Y;rA4+SNizTP24AFvY)@JZfxUDxs6YpIOlHJ z#sP?uR#BVoy*|!DpE&S_(IX#XHKR+vWx@_UN*;86vC;Ys&lD+9T?PMs&0w34t=>Om|gu${qVzU(e+er5J7Ia}x*U+(+88 zF)hPp>N5D;fhS)(J~@=@EQJr5MgprFLL!i2vk|)oBP@6S`~%-GtVOD;2xnJ}#^lo7 zDXRy{=R~71G8IaU^+}MCPAkiamT0>VyD=WUort&$d2}J%^D&-U!7$JX$qiR!W6t}g zfqYzaTnFcU)aI?O)I-*dOXy)oaL{I*+h0 z`wJy$tT855-)u-@=-00gC$E8TV%KsQwf5P$;Vh3lZa@W%$PJml_+lzq|9)hXqc^KX zDEt(c{F*a|uk_tG>$JxrOm`-qRU#A$%b6U|GZVo#Ue(f4%FoYNy?Pbz=dA&uf8=s; zGqT;4D_2x6Uq0>L`_TiX-nh5R{!O|AeHz(x40a!k+8UMaIfyT{iu!yZc76=)Hy!&? zBk;bm1A55M+t$%hjLX4GLOp$be?oc`6%}jkcQ*L|0VpoLzaF|-CbVrC2j+XJtpB$- z(|-n>{w;-@F%Bp{Jx3U4V;w(wip29NQtemMa+FHv`1p9%{u@+5fYh;9P-o|x+H0A8 zN9cg6pXKgV3S5n0)T=Sl;V2!@f0uk@F_!yy>a=ucD{eTf_H8tqPR2lyl^cjBu?HZ4 zXZfIm!N2b)ll&u>d!e8;C6Q;>!PF;W-QmmrRL>h*%8?zCSb2 z1pwp>=7D8WYG}BU>zN@RUh=)HtBR)5!!Dg?@89ei;-h&2Zt9nLBiX*bNO;<}2%Zy~ z8QAsJwwVORS~;hNGTj>M}M_)sHrfRXzc*&Ie_l ztMcTJLYCsxYkNf{X#Zcm@yO_0)*)lZZz;5vndeHJ1MlhssiZGkj3HD#y$=<6$r z;{gP`pDCZInDJ5IX(|B|L;--|TyEt=LkYO3T@bLKyJm%XmPL(_2Y6i?<)a+tnwI?i z^>6m>G3A;f*Qqg%pDHvHc`6^WanID$U@6n6$eYB*7#yG8ipxeNM8L|2q9hyF_PeVD6hcM^Y+5@hK; zwXtnmLKAK>*#K0IOq2df`lh=nlbf`nt2Dst6r@)mUH^6qOP}du8M* zV;t(bHY}wYu$4MRROa`a?>C`QN|dr)cX!!$r_9$6s`F1ihP*Kmk10c6GRrA+uUQv7 z+%-p=mylzgBfhto^Lr7V5lU7$>m#dz%SojY9ZDO28Bf|h2iZUpI->&ac`Bb1Z?GF% zTx+X7OXf=>vRr5Lf^{s4a0-H^+R$~(i$FMQGSA@;VhkDUpYLwky<6o2x>%7fI?dRd zilj%+4qpj4BQ<`05Tu!OH#3)CNZ>xjyFxD;+tCt3Vw-%I4SR!mvbykXZ<8TQo<0$4d(9=wQ9PUz8i1a!mWRVT)9b*Gt?N-QfRyX!&Q1z%C$>H6GOsC; za}8js1HEDvAq&~pa_m{NGX4u-@u^}=GlC874Adq2_zqFpQYozJ1vGv-}GdV}D?98B?$PzRs!aHF52;3kTH z8@m!_*yFG(%tX78^FiTzP3QOYqG(vYCtw?v52*Fi3i%EXaIZj7qm{(rjUz z+${I?)^pqVOPLa4U;-8|fkW~h_?GXkJW|bW+PpHN;m2cb4H;#So2+!sPZ2U}hf0ao zl5|igA1JNIm)}$==RJoWom^)QeUQrJj6Fb#>z+T{XA4s}%)DKd%4k|Yj#sQXC^JKU zs-6BF$&ZU=|D_rd+z(qdNKf~&8i1w**+6T{DzDSfG?WTwT{J4#L?evw}n3O{}k1t&>s+VUBPCx_PH(H*PlnBRIB&9fB ztOK_r%n?6Y64Am9>-oX{{PZ*ZV!i3>e}FEM2KnWf4=NL~z@~QQZfml6jShD?X|}uG z$z=<9g`owRA$jH}bFm4mT46bU=$fZ6y{qjZ;XcfOz0{^VxQ=S4zn!wVS6{9afO~BT zN&%Zjt8{j;H(bEMp#0V=DLFkd;pzvOADObQ86c;gaGx?7!tD?_?w7UN5b?y~bKWQn zzaAyVhJD1lrS3Jt9Au1DMyXwFtB4xH06|gLGg1;a6b>=_2{yqWip6)Q=t>V!-d3CB z^(?T^8ak1UdZA(}D~|BI936*gr1<+>B@Mp(0W~~N&Hy!fO4HHOkVhlhRc&>k(uff1 z+a-`B`%B+a&DNlC4PQwl^=qghmHv8iV$<~3WEU@f`Qn+0#Ga=RuK~W|f`w};XW{c3 z@WZR8!|!i;o^^|&_A*J|9!koSHvTv{Q28FkudvM`Bi}75gG($t}E-BUY!|8qzd_ zx?7gLPwZJBpEHw~GsUoCLQUN!(0mP;W?3si?^5>+-Vj`5vb$hCjP#{yE%AO&Q99m7 z+zmR?9g`)PXov6KXNPl&@;Xh*afKl#S$C{Mt@ZWl$)Wu^odkMX){{r>QXx+UroGui z;LWSH9OyxcQq2TK9$%3;$4L)6FaE~63~ZHQ+BKIuuX5P}M3mLsH0g}8r)-ve3{#u; zWHtq-?oB6KwmB$$lupZA%=qx0uePn7a_47^J8@7Of8pa(nCb*b5%2Vvh6#nO=5Ri5 zi5KhylyZow)JhmrNR~C==bi39jR6D&X#R$&sY&{qvIIe~YDkQV_X*-C)?DlU2x^dAa=*8`TnNB;ZE$NKdn!oa37$|Hk*4) z!%guuNLW4yZ2 zZaZor{`X2)2T!3cPiY@b{lV~29O*f`En?;%4L=Sb-OyORS=SInk7QCW0@W=J%#JE& zhlwZ3$n?OCWGHvu(^3d^z{s#=-k07|2Ovei?)UBUkr$^M9M3>(4mMY2j&Z_B@j?+& zrgq57faadUD^$nB1pDPeKbIUMr={C(6@G|Q00ALDlFHc=47*h`+>P4BH5}k&wr)(T zmVgx{{vlCsqMb^IDSM#AT^$rcYL;*1){pBH53VvNB>7GYjnDLd1NXJE?Mb&2Q)@}t zLHS?cfbaI5F{cRGomjb!ZbEWOS^kK?p8&-?YyH>mFO` z&j9UZToI0}*o}?`G#4jChaMOk81_3&vcSR{j{kZa65%wqB&<~Gfs+O-&UyW&Tl@a~ zMC5;BHva$jf8WBb*e(L9wAEmBafK(ie%`!4AG+N0Tdio}e*JsJgg_|L z>4QXaUWB)wXMui5NXQpIqWXw0cPwpACi9RL%KJK@}RA!BffKC8s5OuB%ZP+WdY{C>o1!2hgL31sKj;JWNtMi zRCe~B<)>W&-0aVxhHrjaM~Q_pBIO2o+)TQcEeW^BG#11CDy2yit>HyRL#~`Jhb=-} zabwSH#ARKBKUt|`0_Wsjjzye4p(WM#p&rY8hnpJH)*;W7W2O|Gp1Vl}+R_CG*STDc z(0fHweXejRdC7DtT6S;@NCiaz9#kl_adslSm51uPT54!7HIz@zLk7K*sTm}Gi0-eM zS5<}Qb*`REyA)sZH;Qm-ENnLE-8<9HzqZ~SCZk4_imajwTN^xXTFF;$KT&=y_uXyk zn?a_xhg2PPtXm_6sd{0uX14%{(#OCL-O|=njs*Q0w^lp-QsBP=ZiP0eAA-L&_cTEo z)Ef1?&MlM+qVe_5pf|(UKS`Fp_mF4CV>W%fZpqU92REW<122cG8W;J#*xKk{CfV5+ zHTjy`|6(U6uwDHy?Dgt8s}PW>c205BSxs`#QqoW601rU@BZtkzH8PIz8fJB)p%VRBWmtz%C>H z1<6>qz(5Al!#RFRv%(rK9Iy{MVWL5)^2*-9W<)fAHTpG)6}mTXnd{GoNhA*qXO_r))Je?1m-k{(Ym`-s2? zmbi{}Hw%tP!~9mrrqF>Rp{Ha4AwU*w0Blvt5x}p9j)Snd5oSzzAD8F%LIO!#a8!mn zDU(b!_4cRQBrlDuX1xw=3mobtj%o-SWTa--tD1Y~6)A-@W-gU*-fIDdxKq+Jb=Nyn z>#V9xF4}Pb;;@Gq=P%MHtG20#a1{{EpiVG75wFJ2yt{A#j@Z_Cq(VbHA*rD%?Q%1c z!*ePZ#ft%-ED15(-FccTZeDr=<^x3j+Y1_E^tH7$+9Kd(-Iwjq#b4jnEuf``n3~3w9RBg;#;+SwC;G!@0s$ac??T zp#mwiR0MxI^snnqBL6LWLmiN$^Wi;|msd4N8Jcrl;R$G~ImLaD(8Ar{Z=7N$^c7TE zowW&C(%>1N!Ox_J=2U3|WqE}d{w+E)pRY~$>}R{qN3L&_aM$$*Lz|$?8q?wW{=i?q zQ16cn_p9vRjpgcl4~BNNnOV)l%f_~|605$Y*s*x$n;3AIsO<)}Ku7jb2lM=-Os*eX zG?tp%pxldDoz~cGyt9T6P~f=<;i|IY7~8N6qhsm*61k_Zd7{DZ3HTM-`bzbHHYcp} zJ2iKIC@2g~3jQFpB#TqRCT3+H00dNS#)L`~p8oUdry4*aZ_V-Ct1cynfG^cvuklFU zMkxN0(lPZWn9?8Vj`+m^0f1jOHIC3d;)r}j(f%is%$D_)Li%DnyboAqOs9?k1|FD} zuDgKLw5zhz)+V||o^94Szu>D|uO7tf*~)b#>f z{9nKWx23t)W`da!3$3e2{`SYmS>rM(--;K$d-jJYFAaCS2A$ws_x2`)oq2YIRAPhV z3k2N+^m}WF(DBOgVSC8QNb`;~z87nJCGo9fgAB^tE$Jmn!`V zRhDKaR&UYMRQ>B5mo<=As^)uS%`1Gc0Vw>mtA%?u&nIGlB8dmE5cA(MJ^nG@o}{Oz zbIVoH|HKLx6m!wB>Xj>JI})T)^7RuRa)UNf$~|^H4m}jhGI>%_Q4#0<%BOB#+!Q8HQ*Pz3qXlyCG2hFLyzNp4wVvghq5$>dOOZZY(=P@7nf-q} z@4vWEFkZbQjhxkjBp~4$i;V^+r03Flv_Q}_y8;G1($7y_iVxs@8yV^;LTAL|oD1~j zA&W_z7&+IS{-kxv#4slCo{938LBLJPBlNM+*X*uFJ9|%Pk&(R#UoqkC7fRg(TUrY) z2zV(;2&8Ijs$bC3;;61Z)wtP%7OM<;Lb{s8PjR&sPxmN=Y|7=UilA?Vs2ehxliuYf zWdG&yhH)aJs}UNG?y-ZQs+0~gaVe?+`t5Zk z+44I>xl9rOdcdqBQH*S>BcGV*uMZ`ehkdY&yusg&=^&e);Rj_o0*$AiU|e0pZp#WX zGNdGWwHj5`M4#S|L_AFHk@3&Faas9nqA_|qjvDTGm+rSVR;_#2X^pobO1Q1Me4>@r zo98Muc~&)Gc4m0k*TuXZv$UXNE=ipg^76<1I^X__?Mv>&8lAywsdMg zEzRG?MVxf0_@Z{V=+Z(O(?Q#e34+A4)aHSDb7 zR4-lXuL{VL{Fx(U4h0uQEZWT(#Qg?QJ#1NfyX%cSVQSt=7+NWh<9J8Z1Mg#W5c-RJ zHRtVZrUCjR%OW|Ci$QV%`~CQiyjW=$H`H}7>>wA@4>N0V?T7)>XZ>=VpSwKI-8$>$ zQYx~Qgk-(h^--a2+1d2l13eG9^n2^pbI|qpAsap@9dIZ?M#oWjQKL)>eTCHaTmI4c z+LEEIhOdbC9&Lp>+O?SNg${yF023KcJ&w^`wz8GqY_R8OMSu2HV)W~!o^7VJhb%l> z_0HUFUHbz7j1qSYKFuG*K{=q+NDgNJPG$Pa63m=7-BdelN%fnss9WxLtNj~Y;V2r(KCl&0M3&O z{k#@M2639NLp>;A`Qu7|h99GErw2dlkuy|w5&~EcqriP%>t{;I${yU7C1quP-1=r$ zaD*O3i)(u?dGkXCHy*oLZiXy<3^CxR((Pyl7u=N?)#;%r;L2E^W3O)jFrz2w?SMWU z1eFTO_-imVIv~#}oST4=O0fns$8I7>gV@b1ZF*Krex9h5fC4`q8I@BvWp#{x9(c4# z+LjJjoPU5YB5{tE953`(xX!fbytX6UrJ~b5x#mKCh|`)M`-+eTRJjVUNpS<5nMB1>^0^7OPD7bPb@=}74MnXmu* zbOl>KT&DgsXl!8u$p4d%-!Geh@fNGO8dYA!hkRE z_YsajZqDdRmNUU+=_a=wjSy;-c(pSDPK^5f+vZ|TAM3f?Kh~p90$im1GL5TU{4Xgd z|A(g{{`2;EYh)ic!d}Cbsou{pzCGy}u-f1T=G~t@9T?#OcpCg2ll>0GM-9H>o|VY^ z+i+qU8a@h^`&dvb_=nTx^m&)qzt8z-*!|W>%m2hoa0BswPuTwFZuc+CrT?lQ|63cZ fT?IR(?g>r9Z7^YHo literal 0 HcmV?d00001 diff --git a/tb_plugins/profiling/tb_plugin/docs/images/diff_view.png b/tb_plugins/profiling/tb_plugin/docs/images/diff_view.png new file mode 100644 index 0000000000000000000000000000000000000000..7747c29c076926289efe1c38025d98c225e6351a GIT binary patch literal 232656 zcmeFZXHZnpwl2CrkR*~skgOyT5D*ZMsDg+j5ebr^B`6srhXxhNpddL4lCww-4FZj1 zk(_hRp@~iRTY&qVUH6=`+gJ7K)~kE>FS@knT62!^jr5H%+yAMOEHNPsApiixa*w5- z0l?`;d|v#E;Ggv(OC9hJmi;r?2S9!&-4XyW0dms!pF8WVj(NoQ87D}h_S<=RM;|{m z9MpIwz**$U*_$2hZv@BIz3`S=QzxqH78v<=W)r0I4<=7t;!YR zJpXds_594VDPjww=o@ABk`v7f?eQ`Erk!qw$7wK&Y0(!O+v~)p`%=tbufYd~FZ{JU zlNkx|fxQM9{+F2b^=Aj50!aQ=nEzG)CQba`rhw#KoJa0c^_lKH^+j{rTWLbV%(vvR zufu|ZnAq42rk6ZY=q)&^XZZeG zw1Cb0-r}0-9rHM3c&_CPKZ(pvQer6FRO2P8iiZ)auC{kT=r8;fi2D;gCLv^0LJ+Az zSCk+Dn)Q0!C_6#F<#A_kqR_KhA_K?Q1{E+((nY9utg!Ry!Nb){GuD`NH z4`$xwqezcs=bZ)4y`^Kfqq%_kh7_Y-Zxs<_hYBB3vHN!+vsNjZLu=Vdx^12~25Op= zeIYa`oj7#k71c2k^^-F(ki{nIc-I&xoNtJd?RW)^x~4?FngiX%TkAuG*|L49YZYR+ z@-Fo7jmhYc(`%?_lZ5u_s|#7UfUo;h3AO8BW@7NhTpVPl`o-D7XdGY2U-Fa}dS^aP zr>HzZS3<-S@*#`{1{lb3`_})B)A$@O1tkR3*U=%F!{~%D8aBha`GJ1f9Y!dX+(Cg| zZ@HPXkxM}=dmnjthG!oM|G>g!i6hlD!>Z8``-cv44t4i^OY|D5KxEb2s~z3cUlWH? zRS_asr(mO78|x?I)XuJ0buc|DL0|e`e7>!*nVD9{kcXDv1r`**gUleT^#^Hp6!SVp z0RwqmOIcW-po+@SQFCMlPsjk(=VuW$+eEwryI!^uRW@!u7X{CsDzB8 zw);KtXwK8xKQr^^*Hl{6N=DI6GnKUbSigeX!!A8i%7=UA)|@UGOC)H@79Qw~>b{AD zGF)^w*IcX3SDiS-?3Mq#Mq==A5>$K{JtM1Pe>9Lf|1e5=rWnmIVs&8PraiQ$?P%ps zU*>*?2D&d*&qqZgDp)PxK-qBCFFwP)KH{RPs@SwI^MPlYZh0 zJQ7S6wMc5#_N4Jb(T+U3j4oHeNxmdcc5G7MQQT#JdgYfB?a+c$pVp-u$n0l-GTN(+ zk=ALf437HLz|S z+e=Q0Tl@NLZJFC;u)ujQ zdAz-P&~2Im0&y?R-wGCoa}@UaeM`8GWIWCleps`$nDA?*DlvOSTXF>jiGrGifIV|) zI6~=da7$O%mN9IA^If~SKnY+F)uK=XJ4xnfewQW=l=B}`NypVi+Z!GD$fq0y%X$11 zRM~HmE8-81SXW`3y9VUC_gwEj=;cXCb2NW`iI)D|Wf$o4-k_kPW>0Dax`^dN!m+Xn z-%ICJQkK2FHt3n6N^RKl)VsnE&1e-d%VoiI77GgMNG$n z$Re0Y{9^gh5=KuI1x`>=-QQ+PsNW3swi~`cudo))L@Jl~E{E9)p!QIHt=XCPY>e1+ zqH&O=69?T2o0g4K41U?R8^P(p%8z=E`cy==xnp2gV#})OL_I%$jZwb&1Ss^NlUwAK za|?~5=5`o?EGYAYm|6YA*>V)>pISh(%=95M7E-qkWmC5m@tTw0g#^rV(?S*`p?7H# zkuK)IW+4Z?iKSbSadS@0VV^?PogKN*3tqUx0Ur`n(2M)2^Fb~NN7>X{^VDw^ZB_U( z*_PFH7)7D?%4A^#dKjXMxA%=-iU2Ri>hAB3qLbDa-O*nbZv(;-A9PFe5s^Px@a|YY zTOIujacUpqvxC1<87kzXNpfb2rG1^UO`p+}tU{(38qc7eB+B=JzF)n=0!;zgtcRq| z&AFva`E*VI=FOuhHx|&=H(%)|T>p&hAD= z-fCuSQVdy1=;hCdwFH;-+dwq!rux89E!OK*Cko3z9s3=?bC?jfEmn zGJS=9kqh%pos#hk4<&7ko(I!V7I`QNNH{)Rtv>EodbQBH-x>IYQJDQAee`qY+~CM4 z4cQq8RdWcX`Ms`L%VNl&Y**lYJURx-zLM_k!2ul|o~~VZXK3Hk?RI zZY>j^9Q>f(Ms<~tqYE#PKNwMU_tls7<)Kgs-Kub$r>b3HM0dX?cN{X zcW>v6uo|n&y05QiIyYdcxHYGdz4JAG5_{L$^=|2en7FE*pvwm+SaT(APGuhN`+UM# z2vEV|g|&`v?nKHUKAFliERUoZu8mbOn#t7cX2BPzt-~desle`oACRIo#&MVvZA%%3 ziY-|Iw~_>4s|ZWSV{hzFV(F`^Bmip7<$~fpl%89RQ}eyH_>P?aI=KBVg!Wh~XWKFa zo)i>fnOBzhD5&E4UX_L8bIU4rM|Z?-W>;&e+J)8FA+ww((a(W+Ao{v=Y_FEV;=ZYGJBw?8!LuprU!q#p<`(Yj zpR5|7EnUR}sCk-wb%1yO_1FXR*qUu2Rj=hjihDk6I};nhIgZ7B1Unbp<(huXn#!rc zEN=21a$SRLzj(&M>jGNM-p=Ti($x9=0 zmv)dbKbniU-68JBoNX$0tsz9DTFRwo*bLR-Q#6MsRCXIKx8Ll?uN>4`n&WDZ_Y1k( zRAgC?RKVwTOMawl`TbJXEAukMFDp;zK33h{hC?>staa(Qb&c=9D}N{WLq>|{*89D0 z->PWzx1;L%H}iG6FY2tx0c25QLLuTVD4(E+RFxAqYVQrxjLMuQpF!2w@VQ}*)P zjDF(5K4+1Z@95tSI?|sQeubta1WGk75&Kk953-EqZ!?n++%a5tze|^-QdfVEyy7HVKj^(Hc^dPEzmPFEKl zr)|q~?IC%({2c+G{HIEu9q=f=wC(GUWZp{>|x-Av{{gzA- zVS)w_`;>CH)dMNp2~kZSQ?)L*D~2Xs(T-9GBAckZr&@GdPFG0M-o97-=y9XAN#njkw>FQW<&5s=9?s3?9~P8G z*aiD>lSr>g>eH2IU-pX)z3m^{5VmC*^nn$xaZlDBJK%7}Z5_-IR$qN;5wdzgf#A#! zfmaf!h#Oly4}C?<6lo43!A9uivP{hHwBQv6F!C@Y#cce7MKeB3cbACOM<<;Iu)dNJ z^j7#6jNs|8X3AEdM z<=pI&EpDAtJT*75D6^Ka_{F`&R-3LUTY^IW>xD#^G$v^uwfeY>Vv^dMVhYh_8}cwZeo#| z8h86@)Y>*`qSJ{ITY(uFWky8^%xhq+c90s}9URz|43!Nd;n;mcg_0j%%L5F4JAUE( zRK?$xml>7ta60n&l>Ds?%Hx&~0^xh%Qq%c~qlk9hjn##U2&_QyPUPMben8knWp%$~ zC*@X4-^s#Kk#V=UnP0(T%QWjt1}#Il-Jy*<*C;t|Mf zZm5hg`D7K;%;MyQcDO8e02!m$-CM(N-4GQ0O@8~OdR-%%p}viK?B8ByG;MOOhb#-R zeW~at&!x0vRjaC{TQvs~>3{eFqi)*eRx6QzVv z8dvwi8>Oh(>P(USMg`}Bo`h9MWaH0*gz&@arS5>iN){nt(B(G0VFOa6^u&kvYc;l) z8if6go=;??+4yeNpv$pLOtStq%f==2YFCO~yJ7b?L}<}WX_rg`6!pDrnUWcZe*U}7 zBW%Uf3AH+Hc2FRyb-_aX5>UG8E*`57%}UL>|1Nd8!}pQ>9aJ@wkri~vOAjTSQo#o% zwK8menkS}=9XPuMkRgd(PsZL}2lr$@I>!hq< z_J55lZ%@`=9)QhGwKil>V?CrEulLc6^1O_@Fb@z7?XZe6cXHzd235>%g|lY4v5ea+ z&ziZ|saw1lbUik918hd@yT1!k5LkGi%2`TpA3cI;eW|%-@rBetYYO(V8njTkXzna( zkTp=p=WR)3k@-pkb%K8H-58|(p?9K$I-1o$x2=hl*%@sg*5=RTvqk3ZmVxDXOsnUZ zctw39a_i+lMOlC8uV@fFgN{5d=|>YM#i=L~`iBM18%XMVh8EIngHFoNO)q%4I7OD> z0W(>dXyp6BPud~{h7bX9dnW<0!*DZATV-whf|5`}{a60=cyHb#?Ur+7<|w^b z3rjlEEr3F)XI0|Bpx(7Y%gYa&Wz|MX;A;a2r3%S$-jl}okNWG+N={_Lb?ir9g;1or zw!Tm~DSau{!Y^@=Ua-Oj;I(#NgP0eDM6VT8)w&{_Dm7@8vE3q4C$2i|Lq%1+Z@Xxf z!f1%iH`ae!A5)gYY9vd?HtXNc+;Yax3B7{5P;MEv z>)GUn>o12V+SG4Y92@quDa-Z{p+6m}qC|b*_f{z+Z#wJRa_C^iX8{M(1(2U|RI+{1 z5BW8C2gXlI^v29-Fd+tYvoHsk%=L5P@&B}tL=iF4FRhAb%xuj9G{dtFG$^{C(sFm? z*%wehqfm*;YaCGb%MvMjZLr;cKrLWv_gvjNB)!?rj#E4zpeEa>6o1r-(tK_e%~%p# zoH{=_+WPLoKH=mjeKhTzk%3>DU!J_F8eRi|)N-7fZ@sizGM-I7FoEq=Wp@)Sh1c~K zZM4jEP*^f%NiyGA&HS0l5|S;pv`%g7Izriu`?2yluzDL9G!B`g0qCeiqdG&ts$gg? za(D!ElZ&4EM~csD5aKFj(?^4cJryyY+2~asqrqkvm5Sg^N92LMii&P{R(4d4t58&!#2Lz-U6AOHsz*L^NjBrxwNQ?sub5BuT;=K6}zhub167L@yyNl zVNgYzFGZIRC!_A{=ayNFla5#;%>*pn@AUL;c8y-`qd+^Q)2;5&O8)lbw*Epop6nD~ zzCZ~~puDXvcnooR6?+(qYu_!{5P%yoejR4U;4@IS3gZBSZ5X;shpK_Sl+x^Vmw^ub zv96F%cG!*Q1-1Qrhf_1DuU{&TaEA`X-4>eHA)+ILSoxjYj-yu-W$;VT@6=2v6N#X=pR|!3dLF)F? z`=M{j);{R4J-lGaySF~TIB1gGSU~FXm41d^@5aU})&JXlF+`is_K_1Nvhzo>Z|Ye z!CugCYh$LwVKuebRAZ*=%9KhB@dYb4CP!3*oC;#VQe%Jp4sf8;3qAR0c9c^?1U%|= zsOd0Q3gGVYC;@9@1>8EQ1k>K}_Go5yh}qB^<=&GKWF~ah+pKe(R)3Mu0IN=q1h^ct z?UO08)B515WX`GZqOJBNY@kBtP z_(rUL7YFfhA^$StiDz#-AJ4@(931}oCLXbmd^v6hZbe`n!L@fSTO0!qy*PkfMFKqf zlcmT$a{H9|;hebZj(1zLnzs2vl~fqsw>X^l=^nk>m@|0zTtKXpg=U3K8ll1%ct7=Z zic1HWH*AU1ySiNgf~a?w9QdYd^6DEEP%pPG&|4Z`@Zy=EAm*`H|Nfw(tKDx{_85_Z z7@BmW!RERGwCBAXh?|1pWZ0pdr29}CLGv9esjnJl#J98WZT`%p6TBUZIv8o^j;z^Y znW|)YsjO$tvqPlGlpIOt_)zj_O~H)&R?ZDAtrV9wc@?LP>QhqTlpj6@ zD?YPaFvn^9wKQP-yGunR(;*kC6;)wfgOc?8Y&!^gg5Me_RoM3YW7G}B*4}wZf9q#+ zwwo$R%U1w}@ivm%Z1qnt^1No0i=ca*c`_EDaCw$iCF1!6Z80B>l?YR*WTbfvF_8on zg6Nd;m3j}~w4KDrt%W`8*hr(FkB%HA9DdU2N!%5k4quQqHkNMN;0YWk zIC=El_tO3(;bd#WJp30dmZ}f(`VQokrk#`p^m`uHO0=Ju^Q~nnWk33uL#3ct5fbT+ zFR3@a`6Ax;Sz^@>Z7YFEee-d0bnm-Ce1|*zSbkKWEr0 zq80$_(|kEA*WF!HZy zw!ioGnV@v}SO_{_SA;m!irYh0T zLvz5cDj(SKOqu@1JC$T*)+DRFk^tCoSa zWK|CKU^vRHxKa<4<%5$qR6skis}r#{9QmYtJFqcJC3_Rf)s(ad*&8VuP|5hLf+`7 z+huEFJjb2!@r!_VsW_#g`PIgdG@^T~m7Ei94=)`Ly6~C3H2=xB^S#6An45k>-P1ts zQj(+6b5=@neq~DmNqc1h#*Zvb%+p}Qv90=rN=+bV0Hl4C+H(vSNvhTlUoX(J%3?%?1S6(&rl@d^Zt5l#xhaY;51?QA;qUZKS!H#k$3$KP- z7iCQH=wzy-8t=^*cb z9&g=O13VB zS;2mn^ZbCuAUxK73?kBcS&JGk-p4$va<+rt6pyhXL)nfMsVg~v_7cIGr!<)ABXT@R zTLh~;5O2!z4`<}C--3A@O$+RJnhaPaa_}HOU5fF5Pw!$M8uCrS#I=V<&7F!T6n;SE z@C$j8Uh0OwhPJ@)=kkD`j11r0*8yzv+{T_yRbi-c*+8fg50?O`omvu>MI8m5S?CH`}gPm zgo6VF;$&U>>rH)32elbHe1xHa_)qx#w=(|9pux!Xzfl>k`UbrS?wSdI2K-rk3zR!9 z$zLIWl@s&>-{c_%hoHUeNe`b|P2fuCUjWoV8RwRkmeyyr5emKy(8VQt)qNnQHCi1k zviK{seq{rpO;ZwrPkPhTn*}(CEUQk7&oHz45A*sLA6UV)nnpxM!Z`T5%Aif5Dy$wU z{-^Q(PrG02IvqE3K?EfL-eF{#FwYuJFL@KeOYi@u1^=mMUS80y@Zk62Sh28cxHhK) zdEi)hbidka%n~s#29e+k4qlFdp2qtIe)Z-LqOJb2+7f*3S8xAnaToY*xA|FaP@}8Q zq<~)+|Hel>)|LWzEp1Q8n4U(0$2P9WWs|`yqMNTkafazJyknM1hT((5lTi_OlrSSn z>Ja>HWS3Sf0T#B$=}@ZgK_wgJz;x*;_%^J{&2&XCDO+67ajj$tVtyImJw}8NtdIYs z1tBt+=LJk!`B!6p87b4#m#6;s-Rls*+Z%MO!n^}xxqmS^>j8duFzV=t?KFzw0Bt55 zihqHaU)JDvk2{`SyNQLJ#s%WFH3wqxFEIV*zr44vfT6C-L68$0=h;;-Y+loG!n_I# z`#NAo0gj+0e3z^j|pAIi{y7Dr65oT=GBJ8{x^$$#L^|g6&nKlzRai z=i*x_8n75d?hl(6G-ax!rlhB{WoKt!#Lg4>i=6Z*SV~n@wYR??KHm`s*7eOU2DSZN#DVRR`Xdq@ChMeOued|?o_>~NBUrLLbuDd`I z)Y{rQ9v@Kp7uJijr;=PzA#&GZkQ9&3rcR>t`$_>3uy?BfPC&Xyh+3l5OmtbYJcHsr~&L(vvOc~s(W-t2K&PO zWS$17zcmeBDKIHD*fRh4zu@(MHSW)WQZz2)Sao(o-1GMhyoiu1FqAn9nx$*VG>Lx0 zZQ}SDrQGRp6u=SGM;M<*I>_!Vu+4()b^KkrRsk&E33iR~d4n%#1BIU*REKa7U0)HN)eVdq!iT5M!Z{b2*0B`#; zzZ1q_nG7R1gVw5b)emP7U2@(6mo2~uujBgeFB1MMSI&fCwnKInSZ8fcN&*$7lAiA^y_wzn9iOVN%NFe9v43;*h&whWAP86w3yT-1%02Vef9& z&%k5Da}}-uqU9sc$5Zf_O@K$hBFuXI@j_`Or|%el%OQcR-9NaEpie{?fE zt?0`IwE$zS&amBvr{4y?|9iKjn95LZR=?~JrUGiE&h%x)y(33d%eF% zaCalm4?eCXzDCZ(n~VcmMGp}$&oveef$SkS;G;9!s$OHb8~x3{V^pVq``$2WADJ0+ zT4(GQxNgE7Px}zHbr4;|D~H2|)nKbaLAlkxtl#tvk2$3LjSClgm8@!ve;nB=fsV#ur?M2t9@hECNJ%#O}?H+Dp67<Au!8-OAqFxXKC(st9+xlgMysi1RNHt4Dh7>xhUl zGO0 zZl90f{!XxLG*_lwKq@h%&1tl?Yir4~c0@R-$Z73Q8#ziwGFE&qOzXo8TOv7o{9#M^ zCPg?z;rQdX2;8pVrciw{_SrGV>Z7An$dOxjV{aGjAsTTNZ1Xl zZ26YTLHIfFy?NT$3BSJ8JRw2eWko|_o#d{$n)H4{>BtVbs46R6T_|9pR3pGZ!!H_% zw6nTWJyd38tmASJqr&|xIewN+_HPA#{aH7knUg%gLC@<}%~+C#P+G#Ux; zO09ld{Iu?u6sOb|xn0(7c$Y4^+QEcN$H6lGPC5i)U4h;f-bQLq`^iT&Z+ER9Vb5Ug zt{7xvI=7dU^m+2q2YTD5IH2!ZttABd#odkRP%Sbpl$(`A<7-otb=Qs&MQ`Rmm+yW- z-g-w*R6JKJKl8JTJ2<*Xm;nuAmzMoh`{N`}S4=#;Lz|w@){zS}J}P{jcG!IQ3a527 z%e?#Xm%%qb)QXI@hCcO@=W2^ch9ng+O@lC4-U+{VZ`{UI#|9Z+UbYtAiB(^z>C!jF zHv?zT(bst|o=$){PD200Nqx5;W?-CD`Ew3*? zJVDJchbKyPJWJ;CqsM8?GjEL{MtfoU@KGgJY@#4Y+e6#KPy92H!mRlxb3cTO^IG+r z`-iSL={dh?uVBxpvGA#{&T%iMM0K9@R@k={$B!CVh%1wn0e}yhQj|TwyQ~gJ&X_j7RXo%{vLS3n^6@)~7D}vg@7|9b{L^%% z?H&*@3dFH#L0GBYoPy&giQeAys$Pxx@t4#ay-8Cj>Yn#SjsX14#3A z#kfIq(JqYdI_uazYffwQJWUZWtlIq+lYmo&y-{(TobOXKQXLo64Hw z6qIWKrm^kny`odunnl7R;PBQ)i|X{S>{WOSWQQteMOa-Q`d9&$?i(1Z-EBRSU|U~g zKRs?SS4R~IXHVCoDKO1Hp&wAcZllm%15s=D>^jmsYT>Y5YFL0zHy8L(%%WZK*D+Ef zyf)`9=yLqY%XBbtN;!4t6F9&M=oUjmuMr6~MF&PUWp^pRR$cW0401!$_%j;QNal1J zs!Z+e#RR~h+R<9)Xq?Ds>HSw2^K?jqX=BPPY@>Be)QR~DwQOmy@b!FG8{2q~ zrzm(r*7!~$ zU$IgDt3ab}68l?}k(=WJ0+>z9`cDYz7+#JzZI^*qRRRmAGS<76n)LsiYHVnd7f*!e zp`y6-cyIw`x4BIe-+oF9A39dXvJVhV@l7aG`eNJLH0;A~G+d@3s~{$ne{x)5J1_bz z3U8q4=r}(pP{K(*H$-kLqAIc}m!N_>KUl_8A&@i4;v~0(+^}>*(RO1rb>?kSX$^E# zt!O%_GsM=oG)fV3-|}Y2d15!68=;O*NZnd(kbg=Llz=nQuYqxo6C56b-yv^P<39XWzMrLMv^ z2_v)3B%;Yxw6fcebS<atfi&N0EJ1Gml{~5 zz}w59FAfBG3vmhNzbFqsC!A_Z1BeoMoToPUbV( z;MW&V6DF`w>-l$~_Fn_ie_N=54V%+i6GH?`M*k^AB#-MKgPD!td38xX4t)3vF6ZaD zlFUZ_%>ZDKGuYetKZODK-v+E_SF!5WM#`EMg~qN}Lyt#hg7`ZZ8_Qq!WjoKum`<-2 zcZJly!z)8`9lCd?$gDvnru~?wL0=m>)!2>Neeig2A=qJu(uDF4m$GqJbA`CO-0+aK z$415v-NMJBcjS(X_i1 zz-Ze`EZ=Lcm^zwCL2g@a2nPsH9$)NEg*1F1PI&8ah*h_K3!Cjg&Wnd&+n5u}kqVrf?677()U&j- z?CtGs?9Eag;Ur+YUMsp$&0k#F(q+#{Hh&SB=y|l%E?Pl(31lySd^x?3d}I3p6x%DU zhVhKjly@Vc{tJ)Bq;a~b7r213IHFgx$VeuLMlf6aaTIN@yiGW#1BvKr;a;mp#pI!P zl{h%yWf^>ppKW|MU8qE3d!*bJB8J>-C_IonQ=g+_UvnqCg~ z=leI=wmQov2NAZY$|1fiDJEW3RvV|j6W8rdDFY?LkTW`zgX`;UjGh6k z8_x0sb*#GC?D5%(EjTUzm&)0DW8e=lJbd_Y^i7YVZvJqGZo$_uee!Af{JpUrPOoss z@%`HUCJJS-Zck#H-_Sj|{G*OMAbK}KOs)@RF~}nne-qIBI`K8k%7hB5ZV_4hmJ;ZS zop5lZ;tq63QoiwyEKh@AV?H5lZ%Oi<{3-vsdL9EXBnDWg1zIb3JKS+~P+`$}t~DYi z$32`^@2+#15ZF%QEq7cSzFlS8pZg*q8{eeCoju9jZq#YQBgWh-Cntx00L|buYWs=H z#0n^UTF#5sGQ_I$w!vS808er2!iHGR#Ku}Ydhr29jtK1Tf*D5Gw@ZM@JNp*3Iypg@ z#pZ3Yi2+bL$rHt05^)4nml-9hJ;>~H4vlkA^lyIMk6D3siS4v&7Mmz$fHVJ{DKdeC zR&V8RSFotoJOymZV9dswZ#li%49NIZ&q?X1F(U>{QPitW>Xa;d7Kv5&(%xdUoX?)p z<{6pIV4>lf#>TwHP}|sJxx!OxXz=jK-Zg&c(z0J!)hV-5T!h9wwJrG{&YfHE` z7Vk)NrNhc}angwe+xuu*z#Y-2G1DxGvPI76qm1WGE0nG9`6?a=6A%&t>w8XVj4;q< zZNO}EHkbXpxV6-l1`5ANiJawq-}k+`{~~#_JKM1XP+0ZagTM|&SuIOvD;;I-0S@)E zO>|wnXxb4Qk+m5`3$&Le)wnuuPSz)2d+m=o>069fSE%CFul`{77zZte&t-cyTG-Zz z4JbEi3b+lsR&iejcK75369J$UdZ?SuD-@X*M;~q{K?QKFIVhd=AJ)VT{$k?ITj!Kd zF$NoQmY`|>8aS;$u)RELT8}VXH9B>uyE

K!dJ%Z(W zp7S^-rjpbkYs|`ZmJw_IpsoA$7cR!-oL8|F1vhc!{&^~nPn3q4`|^@{a=uhmOJNM; z@W#*2J6Ge|9)hulaFgeJ&vPZF?O>xvg8U4g#**SR)-Fx?moayP0^G&l>V_iDsX)vt+$bQ~oECp{4Ns*maJ-9$%;nP@* zo)HKCE6dqZ-n6D(VvyPAKG)H>+s{=@TL#-;=S?w9JvTTwB(=zVo{CFo>5TnFKYkA^ zuTJ~SdR##Tpm0t@^Mxu+tUAk)(xxNC3S?)#qkDAJf=^x{`P;WUkHFupn1Kfeg17oV z4=!H9Ty`&JotNE-`5*SdlN1u2GN49ah@D*8E9Cs~; zuXKs;>r^j@Z;)Yg6(RWOFvN#4(b%{4XS78A5pTOcCACN%ws`9z+E>ep+<|4Rk zy@P3m;N}H2SA7{ho&7_5H!+L$!8wH2PH**`p}&e6t1g051Fj-VpSG<})?Zx~_xKi; z#NHs})NzcjKRL#Lk<(KCPwC|2)5C&=eL?wD;^1TTPnI>dQ2Q|GB==Lvgd)Up29YVo z{s7aBv-ZI7Yw)!?FbKz+l(ab{kCfy3{!>BBh*JmbKyK<4BH{%KK#eb(Z4?3;*E0yfUl^6Gf@(AjGGByi~NB5={A7v#R`+Yh;eG5 zl9CkoE&?j%m-4Cor~#|&abNrzt^iy&($g{NORDu$UC#jp7ZnFtFDaMP<_{4NxiCJ| zmkSOujK{|DVX)!Pe|ojsLp&xAz`p1C6ZWRC_f-Z!UyW0sjR}jVy^7N@g+C4LGlyw2 zxdvXa_b!{q*ua`&J`-S}`(9uSC2T6gH-m}pv0dZ z^_dUE8<(H}=$y^1;QtyvOiv|F6}Wc;IB@22Uud2ge$Z(xArKOChug&6F$1}DE8?Tr&f7K0n`KOfIR&qM|N3*ne zrL-z&?)x7yblQR(0<4n><_Ji{h_vFANK2j=4Z=vGE4~h#eQ~33BdW_d--G# z!6(|Ck#AGo`#hB`L0A0@mw2eSZ!rCgs+q&FX)b&f6rY*R2 z<)lOcto4zetL;^0FLICFmEK#QWl)EF`+E+2ZbqfPRu5coCps9_rkM21y*9>lFT9tF z6OwHYKHO^L$T6Dw9`abVj z{uJwOjDQ$o`Jm6CCnYaZ&$kE7721{LY&+_N6q-iGrOVpBmZmhkMhJ$&U%70c34c6o zB<+L0pI1Hw^HVUmz5N4fsm(#r>4rk?85qoW?RyFr{LmX|iy<{wO(CDmzqOSYy0dc& zpKYdcX;N(qaiCQ%QN<{4&bzs=_NBhsvp531eIc~!5J3jKsl^cA0dEv1XklLuu&jh3()rVuVlC|2FG7X&k>ldWL<@Sp^^91nO>;-2c&CwyYeZG&bZ7FN32 z@~ESamG&tC*j4?moXw~mD;5m|N=#yOiEv^pIIYUFD$8;3-2@I+9Rrig>GZ?Dg#72s z(@I1SAF%Q>=2s3VsCAV??wayL9$a3JG+hs5ozN%e^gJ=kb-8|6@6ET>%fYu6Zf^v4 zv*)1*G4pV;1-s~q7LUipz?{_%?!vb)$}FlY8vlK}HJf#7jFHRVjq$*vI_$ek|8BI1 zxkZ>NX7`Afr-H>TIFf~6-F=vvkT0fVguDA}j14+AAfdQ&p5h`VGm zIK4$-F?quSh|2NVn$k_(FYul0tvQYT6|FRzeaO-x8!eldWIv;*_mtz`zUVOMD~ zV9*s`jd7CriOOk>ZhOJYL3_GIw3cB3ofN@$SSdQatSb83PTTVkaS*r@CB5d)Ka&!w z=zl>Lyg-&jS%h>8oT;T^gR@E=+qR2UJ6u~%)TB>L*E;PLYreMipDeG}i4y?u z@6mfJf+|7~B?ah4bmsBcw&r$?@s?S;AMM7Gmcq5MQe8(e1|mrz(R^Js3U(;p#@L00 zM#~8pQwYeVG)-_rZajT+X5RP!0~EPcWZ3ZGV44aReVB1m87!~nZ%rJEbaYu(^*il< zsT#urPCd_5l>$_G_Wd^DbMjTYn$7j*-+bQjlT$KnMlTdXmtdgS1>sdvAcR{(dhjuT zkaUsaj~%Z@&i;y)K*JB9t7F8~32GkfN6>Nn$8`XK2>xTTUOARnQhcWU$Wr|8df{Iv z(*k&$a{2WdfwGjJ&C74TU&0>dlYTu?e z?O_U!t<=Su`9eE$0^5%n+-oj_ACXY*6ODd5LfEeB$1|28XQz9~epEpk`mBSyM$2wU zoJ3JIx5yP*tvGD?n(AMxl603xW&}ole&8)f6xy^#mB0{%Ttpr$@AHI&hhLg>#`c5P4 zjcTl6JbLYp4sA1R;)r?3@We~%5%vqNK~*w9;h>Tc+rl2lMsYKLtf-1Y6k9kYf9wNb z$7y9l{P@u?72Q!afzQ5Ft3S}5ZYz)!s&($>-U=t-;m~=02}rz_gzAcFb}7gc zrx6}&jf4pccItY#yQ4YA0W~ z4?MPf5JmHrEuA00hi4?x>WfY76nP>7GR3b66L8k@jiV#j)^-h2%)$WgEUKSI^@-Vl zLCVM)slm)vWfo2UTD1mLz*@0{)^RB(i74;Khm!hw^%IWj#Ni6-+}hsK?%9g8k5jVE z!ky){``BKp=<$AV;P_vseY`5(T-z)ZBml~)cv4$>)rHFnwmMoqtL+PKDnn*0lU1|& zjCHLCH8Tuf6?GK5N+Px?Wl_7*d9WVYeB&8b-8NZW9JA55xpH7}ryf~Gg6os`Y8$;< zPs{nOV2i!wv)~W-Wxz^rgk));4imOw+VTSrlTP&95ec$EJix)vG+Xl%C7(2U9LB2E z{&ECb@A<2Zstx2DcZ4Oea%gicI_6+@qBFD&cHAetkm;|4wNJ+Mi{}g?;4fk>{~9AT z2&CU4Y`&mR=OlfIKRMgdR5;!=p#-~1MvuF!g#(zseFvdi(_ZX?22bSBUDsKb?`2rf z#z&v{7C)0E6oOh^mvUD5nosNY0h}RiBTUBFBuiFd3KWyuzu?s(T~}W9TtEH|<)w0?D$qo|$QF zPx6H*{9%W_eQ;1(-<#{|Y8IH)^vON&($IBKi+A5{(dAI*6V@1coWRMLp=dH3H!}|Wu z*j~CZCNvmF{#0*(2K5zWAni2szsXNtupO&_F=|-7pjm(}Z98|#$u?GBkY89?A1BqQ z1=9sxU7+w*t!Sc$nYo1~R6Dl@1ux(Ej0q|@{X)B*HS z$L2;&Q+;pOQ5#AALZ}tmnQ9o)(EX^4BxnqT7c`o#;drV5uEBG!g68Vzj+D;+`pf2e zd!vpIZ8PmGUIT0N0L01SFiA9J+HNeQ}yX^=E;}A`hyQyCM-WTOYmyFKP!7VBG{S_#Mq~Ann;hJrWD3AZhz9j$c+Rb>)dr(KQz3g%)eR?a`|$GXrc!ibcLcY>2`N!fkg5b6po@I+4wS5%8n zb`sl;B|$~{$1A?roKz1aUNucP_CDCGQVwz9f6Z9ClSn(mgW7|a&m5D$JEN(Zk)1DW z$MJYVb|TXp=T>9@-X(ZqLeW$@%x}f2f_WX8@in!%@s(*j6=UsUwdVBVM7@g;4T1OF zQ-2KVR)`qHb;0~2Omn91m`|fHHdb9d)?H9M_;2Jv!v=36X!k8AIf7hChPvw$40e4g z)6>Du*wcGhi*Kc_5`oMO6=9#LA~@{%qW(=32CKsVigUi+?%(VG{~FyXp zx;ut=&oFwu-+R4!e`~#ez3cslH45`Q=j^@D&d=UQzahNU^Pb7uFl?8NIUjW1@j_Bm zTYZgT&^6(S=2#1sa?Xy@{yXPH z4(34v6+L`wM5Wl#s^K+nQh!j(?;&`>9+JY)?WH;?b7F8kPL%YP|Hi~+0G3j z9G3jr4Va?DVa^TFB7%XUE4N0<#8aXdewVKH$T@txqz$@&ph*J!%LOGUld!V$KXgO5 zOVXhIL~QrOGk{V>jKNh;Kt{{+>VIjL4gSUwf8S{yu&unq!#b?fd?x-kBf`L&@PTST z=ht-_PXj=@jxU<>cu>;&0(zO>X^0~v;qDlE9wo2l0LT12+NGw1_!fWd5g!(S_RuN{ zl+oNKp2nPH4Awg|XXUi=kmm2#e5B*(BXs8&-#mCx3m)dHoaBcI#a|~7ArF6yTGD?Y z8w9qSi7^}mE<~q)UxMdW!B#!&I5TNF1&SaF=MH%!bfaor<#$1VqK8dtFGT7%-Bt$IGw=8gyt0A7z2G@q{Dx6!mWpm1LaUA%EkHz>7RMj~ss0 z{>^`?`8+yz$Rh6eht9{bu%B{Zw%cR#W$L&5_4srWxEN<4Ar+fyLm($-F+94!mR|F_ zLj<FO-y#yo zl~Vw@)LXD!b7y9=+Mz0GD4UPyt{UL+@dC`za+g0_`Fkde?rk}C#8Kq zL1*|2o>ixMAocr-Q-G%b_pMFq0{94gqhxC<>?~)&%-XH9?m@fxK{dJ0kKu}EUoN%Z zhGlKNT=YxGxClRJa$5AQv?#i~z}h*T8uador{}CiAW8-)vM?CyyUCGNrgI=`BzB&= z0OMXf6>;N%C5o2DLPAIjH8?h^nIJiAqh3lGTQa?L-+k%%)=*7Gz-T z#A$sayx>8BG*+`L-Ddv?X}KO9sm1_lrY&Tt*`J7>be4GkP&zEx*ynnG6T}0R<*|%b zI@4;p|cO2eFH_2$Z;l;mQJQvE=|lf?`qKx_s-ACYaCQ!HGGr z_JxKKzZaS#Yj@F$2cyw_q|eN44rY%&Ns$YRzkN#+REql8aZfC2)5bbCw~=7YjmUQJ zVPTtLR9JuLC%pf#i2?Nam{x_jM=`7t8QMy}EtyfkJ3JC{SBK}EqsHT;@+_PuRU4yB z9iN;Q1|XU*Qj#_c|s+OTf3C^z3)F2I#ljWEqBo8>ow|ZomDqPB`YfMD6~Ul(2D~nGZ#`Cws*xT$16)09}~8i_NZdWXyi= zVEl6JOO%cfY1ZCEevNKFr%$VV^!>RdJ;AE}AW^P;@qI>f*EO2u1*SxOS_e0Kwc_pe z+~)nKk?_+<$L=p3wyXy7XBjTcT~dR;$|VF-VQh#~avuB;+;JFCz{GCY#+0AK z!w18I`K}9eVwKmes)x)pkEXZE-@K46RO`I4I1h%0sEdgG0t4~|ecV7vZ^Lf=I$#r! zUl40pLM(x2NT*cdsG+$e&3&~KTyAGR&ReyM5c{p}b52{CwPa)}j35q5g~$;@cA|Qk zzbw57Ph=}THMNtPGcOQLjD8`kQE=l+ z8uJll%hVHfxyRN!OX0R`hRPf_&vZCmpJj6I7ZZ8gHR;fm5VSl$d~3w_%Bl&Wp0~c# zkibHNikpUI!s2294}80ZxJDPuLd!Ppe_vRpul!kNbc9KvMp7_E;icwr^*o2o{xO=Q zb5fkjPXhv)NY_2uyG9zV)OMXd<9Ra0%4k=Fz!1XILjQhFBuy|}(PspI^H-lw5_t?{ z@4|^bQ`o1Wl!rx>KeG?-M%l)TYPs>m!<0I)UH;pys1JBECI{2Hg^IKTm zIk|$0f2JTp*LA#DfV?-SX)yEFO?W%k7|kMEb5SWQd*&A#fINILE-%@iF7FS{^x+Lq zUvp`__^OkSJCw*#5yv4U8hQUsma$|?U+juWG=MT*cn~+8%a0g@F%7}4Uw1y<89j9$ zTu|lMCZ3$BU8j>8|FXvIoK?0s-=c*uq%wW2NMJluhj-Fehf5KmZx&i}H|^espgS?G zm%geVBulLOs{nmInu3&?H)~A+0W;w63w3$ z=q&f_IL(ZxD$tj@T0LM=(7(wn_tG~X7*2f}2(2o{q13-8-OL!Q?9ueZG`GZJfIm~e zJD~DIV?(OCpl*Q}3bkZhcP3EKF<|EE~jbzubB$&w@0CA2+jmme1TQ zJx{G%lFp+M3LnypkM&o0QIp%_k=f4V-i{y_5Mj%wwvQYDdz0Giq6^W;R(LqBGVq|_ z5}m$GK>h%7VGbPLjfB94J8aIIhOR5#k)!icF;s5a`>VCn!4GGICarL(2o;j0xj$yv zzs$+R(Ltu)7MA<>h^ytBjztLfq$$v~z_p`5|4tC)eK8FrPb5ih`p0eC2s>O$9v2E@c+#e+Jj;1Z_ zXB!akpo1dqlOGC)Du6!TqD=k0%7{+Rb+tP!HvTT^^J@kK%vo#J3^hP`0nlRJaUsT3Jrh;dTFBWm$ZnoDu4xAc-_;q02!PF9g);eC5_M}LN|OxkM(Twkz;KR=!0wy7%eq!W z$dDLhOA>RU)WIEDk@J^N6+B1@6#*`j?EHqx>BoaMYV`FQ%I#nG`}F)2>}+u@-8io~ z|6+O;w>d^Nk8%3ZprcIILo04LfyQrr!R^D_@ba(xC36rWal!Og+{=eCJ_ymi=%uURTODBZR0PSd9IzcVj9J!} zCyi#2WIU|tV7p@+fWKof|KU&d5gx`-rrHF`;CFFphjEWi|D}R|7tueT8T>7iQFtLg z5KyieqgwYwudV&NUitpzQvw*WR|=%~hd~}O82z&G&X)SO$R2?BVrA_OUW3~TR=xG1FzJje(mEPE*u<3bgndy<0&CpR=`hyFv%Mi1(CGh zZy4A43}(kvA??gq*w2MdOm zGw8TLm@NE9_UubIuYp{DhC!q!@O-1c|FdGv0YyW67Ykd>V5 z$u^Kjk5T%b1?(r$IZRS#d`Q#8-lh^7xfW^NTFbE4`B7u-8jCMBap`uj#$8&Q?sW9O zf5p|WKypN-`=Mblmuf8TDUt8<=g*6`r_eE@GiN3Ywc$9Eob+&{!c$M2{gBg>;~Nlm zA%F%pm>DJJ=z+>`-Qviv&Su{+aJ<0&ict#)JNkziQ?TuY4=T0MN?yn$s z`EkR@V|1p6X!JNQGj{aRwQJx8kjX$Fn|;02<=w6jno*0U*T`NS7t7ATTsLwSZD-R| z*wRN8YSMMLg-=YpMjsil-Sry%&Vm{##q0s|(Cc_NlULo_&sE@)9K_X+Ssq;~*;zQ# z6*qC7y=jsmCwY>9eRk{>`M#9$XP!y;d2vU-?ZG+;DwFArj2eb$XVskOI|^zwjaF4W zlb@CM*ZUlo$ZC;WKOfV{y^gj5efUs!hBiY|jPp!~$rKhKVB)Q~mv-ET5rQwXd(Fu& z9+C&RP~;!CZhLrySCHsv;b^{$R@)X%~%-1@GxetZZn1Y<=fzS4tW#^WXymSpx505qUHjJb^s@-x3CUEtIl@MrrCm@+T@IymdTXxdI z-T8|KPLs{XG*Avc1-Ci@jTES?Y-q=SO zIxNvH(?wCJXD>JBaalD`hHBb`H6{6)?RB#YURz)CUXDdo>_oZd4vl2HUR<`&RE;PL zgce+vZNY5Ge;ySvDsYSV)w(6!z|YY78Xe`~yFa;;Pzsp6?rPTPW3PGqwraUBZysUYdYN#7W7*;Huf|Qworf zupZ8)9bomm-gc$InlY0noSZC*O2YdTFSpu>4@whP*}~Q4)h~RZnvOG>9I{GHz;dkeU?)<>80m_2WS=gH2#9MK5zRPLi0Oq^7l@yHgSyfmhtG z3>QkD0z?_iwUr&T2W>>}25n(`SBwJ#vh2HX6V)@40!$3}L>+|meeV5CWd|z*X)RW| zSq21`4SamVmJc$k3FsrEzTCOc(#7U}e`qi*Q$gotruMbj#;Z?iZ`hql0ejBH`sfw> z@L6-6HTJqZWO+O!bR;;JbTWoaJGp;8-RVeTk>AkMSQWLSC#yqf-k#((yUaB+t5tSQ zEqm--B%TtrgA#iaqI$!mrlmM-lvd7DGf5aKm_|OnoU1Nj(pzS>U2+hS&X$ITm+r@pWDoe&BH3Z&(xIbk989zLc*w?UL|ODDPO(E1VDG zmfD(*Spz=Bc+i2fTBrhIJ$t{=uzoRj=u9K^tD7dEhiI^m71u~cJ31ZK*q`QdkaNsl z-l}{(j*B7GZ+ov}yDX}FcVSRXEgFv_rd0q^+1xa)BzhmBswB!*a%nRn4bfQH+duPk zXneiUoPP3cIY>HzjsERel(&G56PZn)hyqQwV2I^JGWE_?9U;6O5lCgfMcj-s%l11K z0QOazZ6NGb4*8M^<$`ffCp!gxD(&Uohib2a@SI!@vY0Z4mYS3XS)b9=9E0Eq&axBa1n=Hj*h8lD0%;3g z8#)^HS9O`IC~Bg}v@e7icz4D5;6W@q@DGEvX`12O<`=qI>O*p3?IBwEaOV{hqYP|! zjuiE66f70eF;f1t<-FUP9Co9!>6PBD*z@_+m7kA$cmSrPHsS+%e3b;_T?uZq;9+0m zY+1xA-(hl6pYP~2cF4EhWEf=IxCNtb2*YN2$)PjMGz{2$s0-b9~{=!x1HGCCe zgk&xY-C%~oh5L#Vj0mvGpHFoEK?fx{COV7$gGYdCjli{UB>c%@`!r+K=mOM^u|Eie z%$3%PKs4p6wCN0rr>!H|AEQ1zJq2C#((px$m26nNM0PJYc8cOEP`#baR4O?EbulQb zMxV}Yrhw+UT=Y}XcCd3-W;Q?E@1?UuCmtkY#y{8BwPM&~-Wa%VVNi@!o}QX8-&5^^ zYiTsuSgq$WTj?5&3-#|>_;i?mn)e#{c_YYFE>N}^p1HvH^dJHo5s|b8U=uNlsaZ(p z0^`!tj%Ul$Iu#;PzP2Ip@Op6eJc5}=o>%8lU&GQo3MuWVh!^os+oeEP<&&-@EPH^Q zwXv4?+k9(Tlgfc)dK%4Wk?EpWlkGjZGoTf&?ZEbZt}cMDC49z4`9mwmL0QZ!V5OGxzndaekA+?2mCkE&0`0p6UNu%qVXYxGrfl5L*`9saMKj{9ix2naNZ2pg z{=}Z)Z5F0QS<^+gJs7l_sSKkxfrrKRrsU>~hckmox;csm4lb2VWpwz^%e*=)WF7KA z`7?z?$wf{RmtE%!wgk<;E0;rc!v_>h1O-q#Up#;q5c$Rt;(BSs0Wb2J*4qoG;Q$hb zIH5oUnHMZX^OW@mn4l+SUpuSALHFV+4TvHBrmw&+3&9Bdh8w4Cfg7X?EIRGRX`Nl% zL@i0ka>gXMh92m!EViBNK-PA_Y~zWto6bMW0+$biSeHnSBj69vX$OV_&y9^v3wvOZ zg69QawRbuN!~nk&eV2mQb@1R@$IQb2xe%Z_t^(CT^|%F8le{#+-}M-{FaOM4{L|!M zK2&pok;!mg2eczH3#JTfvUD>3Y^VSGN}XY#u?R{(gN7nW39$N&YawgT4@vwpap^8*ZrUd9q-s)w$-6q;*!Ha+gE9FwBevE}#@;B_ zNHvm{TbUJ4%!ef1*ex43sa8MPO~%OW4;E9YVyDOaC-wjJgRGnmBepx08!B~cR8u=J zgz2&W{Vgtv3U;qOacUEWBF1B_C10B*q&=}|0vG8FAWNg`LfDSh*{y@P)|{Eb?%Gu#i_vl=Af6H!s#bB*h|%puMKMV zjC0M{ub2ps;zA{2M@S6GPqE_)wJvF|kb{`xKXigh>J($+kV3l|1_?n6X%dO~x9|6? zbNNcGQeJ!FdF|=!lPBY|cuP}lZT85ezhRIFX4+Z7@_w1NJUX_9FsdjkT>@_<0SSp$?zj^ z^YdLi$VSU@@IMqwzuwic1l<-j*!j6!&#>6Bu82T+{5Dt~XQ*ynb+1)E2}R9?LBo&m z27jMRAvq=Pfd$6gkq7SVNjnRcz00f`L>UJWrgJ!ss0@q1xfZtD>cX}Au$J~ar^h_~ z-js9Y96Q>FWw|wU<)&<$3|col=|C6tf0bGQ$6v=A7ylA;Dq%^B|2@^cBb%DG8mX|U zCqktotjKqxRhHWPOUl9vf*-C_T+LjPxc&z;Oj2^=c!;C`ctfv{3%X< zU%k&!*G^Dl;P5=C<9dY+N6Lj{pPXYaQ|hl)O)6Nobcc3nNMy?D-iC|EO?idAKCT!bZkM|%FbKD^q@|IAY1Ma^H%+y zivj%V^0!}yS6fYpQID0t2nQ?~$Wu=4DWUl$^rM#&aC6=%|Q%=;+HGoL#%b!OUvktkK9l@Hg z^PQ&pSEomX50~T~UwvY~0XqRQIimkYlnl6sp(`sNu2j>*jS?8w32Dc=SrXN*Iz?I@Aat8#6i{th#G06NH(CEa|K=JfiO7 znkOmvxPo(cCAga5kyn!Y&0pA#;AF|{<`m}L(@N7e>4w=nQqN*L?KTQE64mH$GTHzN zHPng)gEa&I3hCo{&>rbH2ySCz?U`BcAelMHRoGVN&v590r$y-Pz?$-MPcd7W=N#Jh62I>=(w_-Mud?|0D4r88->uCdLA?d}esm`*wFL1Kkst z`{;27oE=Id^xQ>E%Q}fU0iuv=4A`)o#4 zD%eHqVwZ}rly1CbA|4i*>VZ1?RRg#ylXz{Qv)k($I2A%OrmrqjvhKw#m-}`Q6Qsf4 zvziHNiPh!9HPUe0`43KhsN7KBTO#LJY|hF~2UUunn-q{Fe=g}a<{S^)?|B0i4xl_^ zv(U?!h2q+Cb_^q?XWoJH$a3dBnryOYPi@6Zjgi~RqpghDH^geaNb%lw{uR2tyF6`Q zv-VCy135a4$jiSL%UihtuJ?Up2r>pcUa6NCguU{ z%zXt2>d}hAfb#2jbt7e-DWf|kN;k7%yJsT~mRhBk(zp57=k0pn?a@cn8F)qSodjJs z|H?6v_s4RM&4i?fF9C*$PZ}F$WH+}7qcr3+j~>qhT2QCE8zAzg$k5v3!YgwkZ^`K% zBu}MV)f-AT9dz@Axn`;wC27L24&ScdJLiaMgAUpQ^`heJ4bbFarg>U9L!Cl~ZO^!L zaL1jzmirgRCGmdTe6q^Rt>Y;q8wd_cMR6N;SYF=fjEaNqijm543T9pAY!}2LB{Kjc zTsTr;K#XhwhkLMI$Q-eij@x~DHD%(1gyE}n);;KcY3T@xvw1gs zdi0@tW}BaiUFD2QVVn$W#4m9zl&ml>#a`BUPr3f)Of^`jl6Y3ifVt*Y#!5k;xXkR5 z&1DK^?l^FIanw{Q&w}e+fcJ_@(Q=lFyQ0$`Q83l&@OdUXl7U(FSI9SzdD*&XW+-DZ z+zWhY{js?VK+~U60GEg0o@qg1*Or;?FeHTr9$i>`GbQ$~5(~^vI(SZ6N)$Cy3m}Q% zec%l+5`eW`W_`j*5}sKes$w(JJWAJ{dK`6d=TvLJFYF1E8@^BkHlZI(=pp8^@wqyHX6%)M-04*-fcPofs^S`bA0JrvuqWnc8K%Dn z2Heu<)Oq_&DJiL%naMw^A*84Y*^e>iERx_1=2;-=!9=PD7c~kj8-@(f^@@!V+_qm9 zHV?O@V~8%A<(z=T4|Z3rkg(0FRw)L$)yd|3vf&mvDI6AV4eZ(`Hxc_eZ!ix-h>{C5 zdrdTe2?2+-gMx{;)}JLz|Mw8>9fH8Xn4mMVSKWiecML`M*4nci7KiqgI$aJ0lN+|P z7B1poPV4mbxml%~w9O$dijKVLm$n>pL_DGn!E~%pI2rLEIKGnR-)g+-aJv8C!SG?q zyL`*wY5zZn`4{RBMBe{j0x?&zXWC3QjnEZ8|M$=yUs7kN*a?UG*fkI@6l3rdHR==~ zZk3Gl!63(z|9|B8KP}b&0XbgDWLfb{;KXMgw>9N@{D-vva~r0I59V-2-FiF#dY(Om z53BN$?LV0eRlbBytH`GX{3Wg8@reQhe=k_A`2wZ@6#U>pKw!Os)z3NC#%Mbg#b&AgYF!@4p~Yv`{@FYG#3NIQBc>D zXZc?%iKbd`0*IkQ`=TXepiG+-SIx`R%k^&FJ^RF@K^a{8JEOeYGjE5`UPTag{r&@p zJIGFYXKl;Os@0|IGfv2U+F?X|-fnNuy7BaKR)0Mgr>~-5YG4SV@Tmtd6nlE09aUF8 zJ?Xcwx^4JdSluC}<1%1B;-c5swNl@R>bsJmaW#$$gQnRS?`Sas(0>emIz0cwEV0KN zuKmKm>%mHZ==)P14nKVHEMJ7LJl9y$`xw2|G%i)e%dRcat8Z;{z;1z(Hs5*6l!ljB zvSr)QXD2^W_S#chP@u#y=i{FC=c$}gdzbk5b~Xfds5X1CR#D!T$#P#wf|ALCk5Sd4 zTnd7vSC-o;Ehj~LPt4Qc)y`cft?ig_tvU5J$?g3ai-#8Dxo!Af>&@x2^13#N703E{ z@`VdpDpTrvVzsmD&C5h^>V;K;G9bvm~} zAwsm%<-Yl}n&OUn3|ae!<_mJ6ZBZA9^NXV&83)yWv~xz*ubFhJe?Xm4D%FN*HKO$R zvMz`h-$i}RZ&#o(OgyH7+GmHwKU9;5F(Pdz)TzpAUu?fE{c?xAOq=z_WBLSREo$k; z>{$Sa(Z4n@qxl zXL9IJL(ZkQ(-iR$>-r6o*LNk727^CDA?0qO&DzVt5@(AGw#p~^LvZYKKNVfwvgrR% zupbff{Q63s+v;1##oShLT$N)7-%=4PJsepR{y|k{>u1WZO5`c_vky}7pDDkO3{1m6 z;`iqIneqi${Is>FKu5ihCV&{WEf|7K#?XBRQi^wCdug>1yA$3HZfryOU)^VBB~V(f zyE3Fy8g|a_(ic)GJp~s1s$yNWFTx%hA9_g}7hbMY>4{iLb{b-1FI?&_X%UQu&Q- zDu9Uw<0i|!D5`F)expH9%L}mvO>PR9)pg}*GC>(dZx z6Fj=NH$%lw=VIE|P%!B(___KuK}|K48-;p|QEB`{PDh}rCz48_|AV=U5}cz~G-V6b zI+~FXoy)fQN{ih*Re6v(V;|+YQQvHYnrjPYYvvqO-<-3J6>+k+lla)?$iBxBU`DI= zzKD;(p}kabL*CR(aZ1m4fG)tQ!R14HyarNVDGjA=p*VOY>wbF&tx-3!+QoOzWJ*1I zTV?TnpK8{nm=3X!?H)^cD?&U^(?{V^SDi2V-u16ja}~gn>j9JRErlnOKXc-&FT3gD>+QGP8ocFk42Gz z5N#WgF%WRIWzogTIWpdL_Nn7|zj}!l-QyHXt-;w72;5M$_bw%Hs)`jGwUQWKv%xbh z6leqUk{A*=I&mI>ZSFFdG&+0$H~H+l5(sOqnxGM<*`!k*qR8BujC9lZD9l^pH_WtL z5@g}`6uGxru$_ov0?j#F9lqY`?IIARJy|hRV9MMjm2NF-`SY$l?@Bf=2C%#wBrH=; z?^Gr0f1Sym&;>^6>~R7h--u^J>US&vXd_+BzJBl%ScNSoh|3EVLzRy~`pKHkT`^Ql z^xE2kBa6$3vorRK{h~JJrcpMgrcoQdJxjYyTUCVe0=8>+Rw-+wpK%0@GqHTGWvU-r zx||X5;9Ul_gj#RFi&X9BT(?#fI2#bxD!9*9Ll!eWzZF>8{9GOJO#CJ7Bmy~N z)m-|lMvO~pcdAgkn$Sf%wNSq)^|Do)E=#pEQ`JFeP}C_@Q+V{1Niodjwrfgpr-ohP zmV#%NOB3{5^3zjkOZu&!l_-5GvCdZqpzAdDYs100E za{Y{G{%Q_df898DrKCUJam`}KarmnHVy)2phxW1e6$`L?h%v23xhGZSTCVSC+58_O zYKhz#RB1P1(-T6XdnNicBvenl$IEr~T(I3#O`wYBGne1T(Zk0UBXanh8<v^|`8YI0yGccE5;V(^6Glrhuw%RjzS|v4 zxH*7omj67|dJbP@Pxuk?>DcGzWDmE$9JOkDLPq$h>~y=ne>M{*#&^Uyd$;G)VqV8` z*|=3FKBP{U)xSSAxJFu;Op9u?Y6M2!y1hd%Lmc^O;Ch{+d$r36yxU0RJA)a;N-Y3554*}Fiu z@O|qpNBz^uKXM#h0una?sxFbUZ082g+$Qi|9Y@o>=^~BI3*2n=O{2t4ZZ-%D!M8?b zmX`*_QoO^5aOqGldMufvr0EkQd}IO0(W3_W?6qarN=dYsbK2KtloOBv`OQ& zXD9YpJxSuNwO9PxN(fV6>~s4LF1gx!s(2;LErn>`EFBMuCtpjMt5ZI0@X}Ii#AQ0( zfoZg1=6UHoq5E5X-a#4^No7SHcfklFYq#^)Rf;DA>ulOTJ-ujx+m zTcZJ^$1vKAndT=VKY|hYXW~!+4EA|!*6$4<%FW%K<=b!QPe+r|B+b)o&bc`5S}yu; zEoaa;Y|`|)U5P5tQN74;IV-bL`{d3lMhuR~O2(15r3}12){~&l?loVRanbnr0so2l z8`r1T3F`8@7KK#cCFu;EEjmS1;Opeh>Pyh3aXu>DeJ;i|`W#Y)dS?4Gzy8sk5mWd)WK&d&wsgUJmbP|C}*C0w24 zM!*tgt+fbNNi(r1?ThHd;;g9ND*4uw#fDwW>2WaD0cwh;$$<_2KQtY9f!n#cW2+e6 zL&NEAsS8VCa|@euPbt=71DpI$@UROkWHSl`H2EInq6)p9(Ao{B==HvlA*|Vo;rl2W z+Tf-q?w5$6h_YU-qa^626LiysZlq%MKNA|< z4d|puDjbOqNo|r>!abQ_i-A4MmEPJC_ax;umJtTFu{}a&$gPpoZNQ_F<#g zB7AVis^ZH>A)`+~hw9dkd0&1unSJYXEIl!kb|3s{p z0XQpA`C{ZO3e0|YQRbNoqn$?R>0SR=iHxB-cB|}EH0h>)lWzbovHzWRVs8$vQWCE7 zSf4t&&!FgOWr$!37qu;~0&DeBDzv7Qce4?{SK)^yJsyn7@L)x7dtG%e2Lm?HnS41- znAqzd#^n2=B)b4bRQ((_iU;ta*j~!ztsU?ASw947zVH6&771qEvadqO*929eJuG=8 zJp~A)$s|3Q46)`(mVhXgBYSj$izlKz4nwVP^5?2qnCHX=lm*?yq@x6 zz&N3t|4U*K9?If}SHwT9qoX~{5+q~4rO8&M-vVUh*K3&E6hm$o8p-+htdc6a3iL*F z-#}Ix;=WuYc)2M*6QMmVgru^k6h3H~^6xn%J#1lb`g*&rXLoeL>q@{Jn5HRVYZPTU z)utkD{ZKco>~*_Apxe^IpD8Ag=pFc$=+&eY1aCyo!2|iRmWH3y61YVVWzvorjO8}t z!rzZ65E=CesA7U`l0SvfNEuMrZy_A2qY4D|Ynrv-Sy(eCo?Y}!07PI3yy*9mGU`-@ z#FpLm=MMHCo4fXH=*AJ|7iUf~wzy!9#87uD**~@q=yL-cO|Fpl^HN|?f&4+n{o%!! zhr^4t3{><*Oo=tst+3H&wN|C=#Qu#gOfg7AQsOlm{YIaJXQdYa8KlvY7-ieDA_Uxw zRiC79rxeW$FZA>S(CJDdckvptWE={k<_9w(DSr~bd{sm%h>&S8SkC7})tGBN0wuO- zshg>;r*GPWkT;dXcH<@TKGDq7OC19Ar zEjJ|$$Y-A5wX#K+X7lk9qg_#_%@z%QUoTSba zrk(4Vx!UO;N!0HGTTX&TOlEJH;kY+TP%k@vL?Feg%C$e%BZtC9TU$?O@#t42wyI`1 z?XW+rADUjSm@;f+d>QW&M!S=Ebt=O#YSG7PSe#&j>}tke_&aqA(?KVdU`YzzR84tk zD)O?KS< zn{xhR#F{Ufugw0)=3-%!p?k@|sF;zJ0yIQY5tC}?{kK&6L}pj}*+5lPqc`0qxxzOObs)Ok+m@Y?9edNOVXNvUHRdfoNYiA91Q?CqR=;!(^@uGjQW6_ckG#q z3<6a(*CQ&-G_!f=Qqa0hwE>X>@26&A$P?X&EJSt?-zNUc`@cNEkUR1GIyab{posD0 z?-~SIks5NLt3(@Ag_wBqhXy*u>Ve$*%7nucv#0@oAez0h4&w<`C7287UG=+C_O<)m zTE2tYXK(`V1AR*buElMa9zyk>mNd<2bIiic$+_V#>hR4QGf)l zCr^WtK<~Haas?UrzelQohQS1Y9}7yleQVw@s$%l&$&jmVt6G7tQHud6aQ<&&yT$YB ztxlzGSv43tE~UgU1l&N})Eh}}6u*uUf;O1Q}kS#O?b&=`HkiPAA|vMMJ}1>xA`?C0UDaxTJuVZ_6s> zngb~Wkra|O_PHgh?9r^&(H|)N+=0bDE!G{Y=rp_d39OyjlTUdYS%XfB(yUf5PRcz7+@7H%-;Z z3|vR^>QekT3&fRT2YXgRO`5!e2DM5}?_qE}EzbsuehfEc#PPUtH0Rs{iJg=kh6k6R zHSd@s$Loq^C*ILPE|+d3zPq<-ZMxC>te7@?lI0DM{V=`QLYIdMA5*lA9Liiafp18` zOGCXKVDK{Db+h_ybbQuSZL4EDSz0B$(HZ19&sz`>|L7|Q69EqAd!g)|r@+h$Crn|f zL7{5>JdKC(EsMt>{yr0xZ-%XlxR0pOqMPNscyr$utF(wj2Zl;0*X{6x2sh1ByomBK zNo*bu_~ZB)G*sl9>?Eq`+{Nbb8DsQJ^Ac;h#JXx7Jn?9So|jAT&iaeNkVywUExE|c zYCWwPBwI3Yz&X)uH$LF>UYE_7UKMwd($hUNMFm`*bTO-r?3XlMfMK=Mg+E62l`moD zqS2N*J^l=kUXq9UB$L3i{Sh2*#(vWT+;NFJU+U&efY|FK%60;Cf7*xhZWm;1P4+82 zxb{Svy~-shZGlpy?@VT%TL4z&?$y7HTk@F0!#{5r^2Ls+;{^;aLc*#ChvVrbE&?Mt zX5P+X+97DTVQ+)demznwgL(^ zB5I409nzZ^U&4h9N{ySy^136O8y{Pro{^wR+4|8};C03exbRt91^^ zjtm?*o1q&URcWMzXL7lcDU4dL?B~=ASm}Fq zBy+>;L^k;C%hnfM`D>tjVXcCNQ^QKSS8E{o_0P+$e?C!^@c8W(HXizaEOr>Qbp&fK zvmzf3TXYY!Mnvxk1U31x%m%K)s$@w0VrT;dU)gWURKPQwOE*2rcFGi9i?(}bmUnQ zEIYM@s)e`viR@AUr(j?0Amk6!F<`3Ly!MAOP53rkoMsM)Xg8cLuT!-AWw`daW4C@A=_?_$qHcBI*IZqakg2^g+x+lmO0c9?D4^=M+V++lm z{7X^Y!`b6XuMx-TZpQgMfjJSi6LrP3?=HVCu4$;>-zD(K7}_+j+9`(7!#0gP-#G%0GXagxYwe%~I*R&P_yU)xOIquu{D0mu=| z8mL(rub3ISO3$CzkG;oBN`{?n^L+`tR%HZsfDCWF;}5ur{bG5p>OIxY0|W% zVu~2vQGwa71kukNXqcvq3?(+3l+?#P`9Um35+2qoDepp`lIKxxNY=+S)L)5sS?dDC zDF_;Vt1Nm~Xl8Tj+P=*IdG?rB394!#TxpBlqN&{?aCT^K#IX@EbeY=?4>$|7;a91F zi}qdp91gYwaJqEuGI2R@MWnyi92f>l@+DT1Wfn6SVSmDl{xM^VH9`6PFXpp#1#(PMEH5|1on9|O!;u>YHCLA` zKAl1O%x*4F2?j1GZzBwv6w2rb7KE8LT=oNSa+M;&3io2EgnwMn3m$oBk!=nUbzp=E zdVa|t@Z$-RkxcG>pGrP7yp6Eb;wM+3>61z4BG|0k?D9Hb;|R5tU@fc__Q>Ks}=v~P9;fqK+1@mPFv+=pN{)3Rf+lmI4?&H%fy zyEj$pc|n*Xu?h>(;_K6rb{tg&7W`6>N-WNyoLw{Pz4$VzoH0u5Yu8VgwA zRWmDZ#enC#rbRVz2W`MCb?zc?cv?1~gb^es>vS~m13#)(V6dUCdO-$9Qd_WPB@NK?iXQUNF*k7)LqyXO)zZ>IWG0xBXV z(av-oH$$JrW?h%U_O)aUpf;UCAsTC604z23Z0+Q;Q^rC(pmJm%W?^gp=4U(W5KG_!g zJj8xf#&en4f=iREb69620u;&li18?|dDSTGbcx!k6g#Nz)oP6~T3+ygp9E`9GM;7! zh!}R@^JOORXTBlnIcdVVt6Dr35yEqce2o7umGEuQr`TW+fN{|7x3I-7X83Fvf0!IU zO_1T|=JX|h)Qw>&$ZBtBZ#2CpiTl-_7X0yC{_24*j(zlLUhRl(d&h~#>SzD#1E@ZK z_edw~V66b|-&{5W%Ekn{2Xli*Un#D8ETWLf&mZ1Qw3fuZ_lW$waffXO>nI7Ed&I)^ zsfGS)ZL0QH%qM%J;i^s!Y2ECzI~8|??=C02Tv(fpWaE!MVn|Am`ab%7RxZblD148KBX=UHDOtF32(`QE=BAp`T|~dRy?Ai2dJrukI9N6pJ-<41^bz}H z0xV|!!XHMlNfMsicCn_hlX#!1j{I@m9b4y?fUnV6clSYTa%g@?Tj?2ueT32d&mSbI zlF5V|TIpSh_X+VNd1G>vOvTOW-sxT8lzCaNUM=zA{FtC|zjt&ImsY%zVo@utlk<}g zePM=cZ*H(H(ni~8)IM%{pyB=fd0!cWov$KI8;7zq`xTJkod-U!-edOHO7VqeXFc0v zwgiWL+-rT-Tux+m6dNZ{UlO$n?uI5aJ2f5#KzNcSc`rN}-6g~ESRpT`&2W+bS*g3b z5!k|={^y@vS=B-k4ae=MS=F34)cTg6h;xO>Io|SP7Ja8|JakXbwIDC00`HXF%tb{O zKewy}U8nfq`r_Ix?C;Ox_LltU-JP)%b`fvk{jAfvc^AvvWA=rFh45S`d}C)gb!-iG zUuOe*xJCZ#Iou;XOp;hFF(r@AA7LbUh;_{U%#dhep!<33v^4u&hPM}ci9$WT@1ODR zyK-f>`(Gh^*j|nH^Xe|7VT=E<*t+LlEXNK$;vsX+hY37x^PwnQJE&2l5J}^bbyew> z?RkGdyHTpX>%yY7Fyy#aYxKeVp!g<(#<}Fw;o9K(C(~PJ50@^r2B-Xv1;9R{!klS2 za`ix|W3kpH_boKVz|X`KoB`pLyH*v@W8so4U2(^bBYuC}=g-9*vYkZLBQDl(oF(Uu zeD3H`cl%wc4AZ)ItNrK0_Q$QS_tuy%FSe5j%m{W8bdYDX64Z_&!^xndp`}Svte@h}+ zZUe{{^V0be;|=x$U!@q+yMa&d=G2k0oeSHx(h5uwO68JuR9R&aPfcPAdY^^3`Rs-K zCXLP9!_FM!2U4kwl?UD$&ULjq-z8X*(qkX5+%%(yZl@Cg(;iXppVTsnHE>b1?3?WM z^;V6&`(^*E&Kn?Fgn}XG&$vr&;R4aJXb$K3E((~F;9iIgCs(_js}^~igB-d5h0Zwe z^}uQq915cz*NW3EiYpSV}|G=Xa|I)W`S5Km7fMfr}!IxO&wq9E$VXb@32|fmR zHdhbn!gwFsC}U?{)y4TtM(!2A(n6W_J0^;Pm+cfIBb3t~RQmH9tL8Eis)w53`-c^> zhHG&*Xb3eZUGndc``B~j=v~OaML+uiE(y0XplG2>Wy3e}eJ6Sa3lPW#F+b-wRiPU9 zb1toJDo&*qe;M!XTtrlr*{}$t6sESrIxQTmuj&QudU)j{-$snB9*ce`hhz%7%wt9R zefPoH?x?lmJK3mLs2B1Fk%}4Bc6PT#^SM;N_TNk5=UBI%f%{yFX}j%l+I%*2FM=PH zeltyT#=q-e`bo0Ix$6G(A{Vs>i@OfemqN)1nmBk4hMdNWEjh0V<|L6=!lzQ27i>(4;_U-^o$ClB64eojUwMK2Te_8-R7@( zR!ElYuWamyYNUo4KK~#m#}2o4@|WaXEg!xA{fTkp*l8!!N;U((w(sBO!CLV7t6n}s z8V8+v5%4hLyLf-w*&7npcP$TRDoojO??Ue&!_h^5jft50LZ-aJ|A(-z42!CZ+8z*8 zz(5HVBn(1yF6p@ye4h1BY?otr}QKW_jK{};}79LO-O1e?HyJP0tXU0I^_r1P9 zKIiiCnX~uWYu)Q!wU7KhOWblvWT{V@c-LZBw{1^q>49p1NXh-sxM=Mli=vrKTUcT& zD(L0wsLMZ9JqN!<2|o^bbB9V-cj>?H$L}fuC&9B7qjJU5Qy(1L$pO;j7ym$pDUgkk zrEVt)ThVw>LTE%)j+k+%G@PAgC$Ey@hknfYt&m9N#F6@p!GVDJi%IE6qxHqE!7Z+M!R`a)ly8UixQVmjcjAW%AqrD>wsI5YT&+@eiV6 zm#yb33il4nW$l98RP|t7fsr*&}r*k?(|Xxl44wWeMZ(ZW#V*d_`jkDUmm z+kSqJu3TY4JMqnH79@`WbvVH<&4+$9Q#&~4HGWK!PS5dh8h0V=mUOF9*e5q)w2peHgOA*6Yhd0k zmR?27tsy5pBQd+gZU^7S9V{Chns)~M&CZBc%Y#+9&vM;3ycys(I{~;wBFD(Hz<dPv5%AZq=Es8?;Wwjo$FXkAFW<^YQoL07a#z z2R|=4T_g1ZZ$Yts;3~?4o32xE10f}anNn;CaJk|sS_w{RjbhCCmsT=IbhR%xA+JW?ckYkTOOvRahZUCsPT`>sc&$R%TuS3VzrC~ybFb@p#jz`+0C z+ysLTjggVN9XYu3)Oq~dQkva%ndxlyH@r}!CeO-P-(rTl#Knrn|JYFcOvIia`mQtj z9aemi;|~>JwkQ`&cqek=*zV`1oVhuZ9N3~mfQ?D(oOOpzVn&^tl-&$QuH8dk*h*e8 z{aG>ENZ5X$r%*t^`{}yN&{K>WjIMW`SNqeBk#Na2=TK$o6t(kmOxd{5mz*v6QI`>E zx4=#3kxu>tU7P6erAPj{ZSTUy(W{T>tm^`g(+8#i%Mkipa`CVdfwS_u4ITA`clXa_ zUF{2!_ER-NS6aualuJ90rSyMVkT4b6E{YtjwUhDf*_9J15;Z&)X0k0oVnf>!Leh|Q zP4KhO;R6St^no45=;t;veXR{y$Di`BCxwG=dB(lLT04$wq}q(DT(O63-%VR1inW&k z<76|^9>4dvaw)BtAp8R?+{E3jo8B-&H_mKuCtd!+H`&uUc<{P+uU+m*#3v<*w_r|e z8h=X=w?;FK-1@)NrdP4RTJ;I!2k7NtJS&5KGH^lplDaBH@j)=vyYiI1(b}%}gMkea zInUy@)z(&o$cmX$1R&2#@>T|J)2|K3d0w0%dVL_M(T2%TE>CW{jOyeOq|h<1*p{OS z%gOIM|9C2j$1yf%+d*tU<6S-e{oqTZGE5nxw#7JA!luqD6LNkkS(t)@Q*M@u*z=Bf zl*c>ITlg#9m5D#qyg*&&50C7j1}*Qe(&3H4+t>m}(!NAwLQo(&0@d8;<8gRj& z>;BVB?ton7SXbFku1(aI@R}3K0&}POn zJcH$r*rr?05LUk-EE_+#!4y}MGJea<0#l=nNm(;whGna2e9Fb}M(9dJt1hCHA2e;= ztw~LZ-nc@F51hiK-mA&KLp@v&NlJyMNz8VtMAk4P^_+%zLy~NSt^1=7trH(CKp=tc z!N>;081A+@hPtTg@*6x~jK-*G2+YX^WY9)!>@y;a&|Q?fT9FNnnTM91s|qZjTG?ue zuIj_F5u`NYW^1*L4yMdGKjiwBA@dB9P;e`>x?%*iPf)!yLp?Y0O&h`A!|3nblHt@* zKb%=t;a!PWS1v}kmfMqkGwwN6PY2a$k(9`wXY>9BPlcf)Jzs~Hy4O^Dul5uw}DgPo7Vas zwp!U!Hh?*}UGZ%A)|_9Z{HV0>P-_|?qqK*xVa+(-xg59K9y@z1wl3+}j*{qHti{E+ut(e`f7!&9Ee+w5KrLdKiUE&Ww^&)6G+tBw$_~ z#~;8>)WSI>2<9oDJbha1;+1*ow{HPq3!mGZwpsVywnOsllOY+FZti5K6pA@!+cQPu z?a4AXT`m-9snYJUyRB|k%^+N86_m>FJ^hbm@IL&aUAAS^&Q$56{?+o!Qt-7=;`!0s z2`-NFmM?!Q{2gMMa1Q;iW+ecl=gD#tU=Q;w8n%f?KN|-#Q*62g={6X(1^|x^v5%t` zN0kl(3RvRt3Qsoq0_z(!$thXl2fnJf1yy}&8(*!(v{hzprO(8C85$i~Qdx}c>X9kA z&qCQbw@I?ff=)W+Y;7{B>-3=^DZ;gh`1f{UUSC0|#cG7Q2BS>;;&4VZ{v8MuM{zXy zkCl2lU^U^)-svpQ6_}#o+7L^Sl@=@A72N||Ng~8CV<80Wx47wG5;$DN`2R7 zy(u-VO_Ug8M#4cczF7Tf<>npUN=)1$8RqR(e-G)_QZWaeJPp2Rml!4=M!>hft=)dp zoHWH^D0*1?mc>9YbI$J##IDJ&iQ?tn7p;8zXRFvJQ98(?TY}85Yl>I28A!uHNVZ1J z4M!~PE9Ts+t14oHazRRd$dm!P7?%K5tvn62IJ=~jDPiP#NJswXH>f0ImPhn|z4i`x zEnWH>C=)RbUdtiR0hc%=5a>wsNU^UyYw`DM84*u;XS~&@J3A{uuobtKB4()JgbpaR z`Fx!%+SZqSN(8y?T{wL^puzAPi~e7Z^f=h89%sh8dxQtXa~P}>bORke9Pf-m!~j<>4~KR?E92PcW;*S-6bkpz(d?f)=t zqX%Gyv>dRy@6E_7;F&`dEbPNSE*YY=nz2tNUiYpS6sF$#))=PMA0j!$ry8VX9~U;f zopz(jJ_pdE9EqU1Dqo7)0)t5l4XYLHND-q*2cJ>;EALUF;cJzWlGYJhtUJHcUXW&o zUk0gzkQhB$unZGmY43;KDgSXv-##me8VLBnM+wJF%kurYE-!VpJwfy6&Vf>)L2y2_xq!0`I=l*fRR)UC9DR&bI z9>HY>pazNq%?AI_5m(CGq_$CxKII}mY<2^?Z_4{{`9hG{@EF;B#Ov;=+*+40sI>fC z|52fJOPpz4oo-jj>?e`9HxWKe2nKz+)nu=6>#yq1@D4-zOAyEFJ+$|80=dvG5FTU= z>rnoi3(duDy!o$iIep^ zSsrHLtD)ph{_tRrRwVMVkvG`0*^O}Z+yys7JfM@Cwrbj}x+!Ni@_Z^mX5&#Suq1UyWe6lCE6eSYp_D+~xDc&PfN zLjdM^4D3``^zm_EVDS&Zan}*Ax*fJmNb#Hrp*OSSrI}M~i21W23$6vXJZ@P zg_bLpD{JTl?~O$A#*Vi)C2_JG-pk>9=Qs;cFI~>;e^G|M49m8>0o(dX8V%cb0w_XJ z2kfRjW^F%LdZ62_3z8cvX;uUtyIxv-2;-`t;noZ!@!~V%LrL04>RD-!uKvVDa<1gt2yZ zC<0TFeRHkj4xREVt=sUEeXk2l$U~&L5jBTtk|;e4oG2c>4GDOk9y55pL$tE=k3Ero z)rOo%zQNz_YqfD=Pky$l$W?c%Vk=s6e=bb<0!v5Clxsa3TKlHS?ZsoEm(+r!@7%a- zLFF{j-o(K1OVqit@rP2HNX%$arsl^oz9f;Tn3c#k7xy9yoyA`irJyH*K$0``2S$Ej zxD09hy3c{vbfLY!;L-p#@o)c8B)cOr%-MSh<727$?4kSvAi5VplOjNj)%4)m_t;_M zg}CN0QLo|?tGj=@CDb**Lb~!bD|6~k>$J0v@-FK0re}(=wCqKz>TGUB+SQ8>)9B&7 zy9+;Zv~|T^V1nz*r;-!DRFPRn&!38ND9h+1>lSjNN_)dpON}WdsAsdyG^#chS&Zt}ICL^lC7wINr9|vDgx1CzoELhE}>L~a& zMd(`UX+yS5^*b@5?6HGl9p?=0J? z(w-BJje3)cdI^vD6Q_tgBBXtMmlhkh3q+U9&lBM(5xil>Ue{8(zlu`P(XsR+S%Wkc zpKN+|w%wn-GLsFj|_^52q|9Zw_NL$hO?ZnhMX~rm{P+niV(m8Bvu}U+c0l03w-EEq?P5RjqCDto_a*Dox?~ zQf>smq$Gt~?9GD|KapRQfN+;`VVoS#VZ){HY0>kcz#P05)p==;lgpk=_>5ceBW9jS zlPWA-ZBrXnX)&X`)n}T_Um`BvjeGnR|NF|=mqNQaRWRBoOU5KOF6nvZ)KzH-dMW&t zV9-*-9}m84Y6j-7sRXRY<58zatCPVkgamo2e*sUm4wg}!`Ws>G(Kg3E#@tdFLz#80 zD@5tMQ7iQstE>G2f)@&nPs`L)sF`?fgQ|<380j-Xy{i)1Ipr2}R|#E)d{V5j>v`9#@8~n|ZzQ z2U^j4ne;y|S-iHeg4|PoRs){s`|*o}zBTrVN5vLVZZK)|1)*|{^D?=z7u8b=XA-!E zbhWdJm4DEN_q-L6V43s~FK*sL-KU8M?rP!fu9L}R_Aey8d^K=}$jyly=0y1HWnPV9 zz?tI-@?v;x;R{BWfzBFv(VsFW_SBa$LKz4%*PNm2#6oo~)9OI>jyfidRo7fFP(y;-H+zMlGBIXKUZi+$Y~IGS*#& zu)0vRPv?{1dWOSZWleEWt6RHB>t_1*g4X;BKmIuuzr_5|Odc`lXqjR6m>;lG;1TST z9fve;dIZhM^*36JKEuXLpK`;pxEAcOERxGSBJ=#WtXdnv^|qY}=Xitg%`d1y$6@D^ zJA74)Lpy7m-%l|eS-J~9IY=(~w^2G<6?);w zXW?0o&#$;W-svVL_{Q`B_Qb;V;d%Vy1lO>K$nlkCLgydka>vThwMMmw%3MAXn?BNS zcswx;FKk}uywic|B&W`P4QrMP)Xm9;+jSBSjmTz;STTpxvvjlGO1S^?XP|4OMj}7QQF}4A6M9>Gs;z%om!3s zv+*X7m1J}9gYFLdhRJH$b3sLn%nti%M8J96QqPc3pu}N#9t6)Ji{8z5c=J~}9Vm4s zv{s4c^*j|3L24}7)oySj|*fHba5tx zUIbrHK17Ovc@tdGlm6EDjjPaH<*OXdgk(|UxxO*ZhVI2TVzob)X>V z-C>xC9*gkzNIMZMP3-9%ZS3yBG>cpxK&<>ETl9{2Q`fv&wPIWYo`Pw=x5po{V;2&? z!D-+Hw*69+^w}mEyRi|%Uq>t7#D8KHawQKNQsRb$=WCkb!OFB zxVJW#ZOJw;Fhhx9jBaAf8V|zPvy?n}KUEh7qlZb}y{$~NnVveu_0VGR)lY|pCNWp2 zO&2bGq#a{=P<6pUls@RE<#K7TQsEJlQ{pUlBb$@^_9ioBEo2x2DmUk4CUlZb6rlD`gnMzA-XkQ#-<$TF0&taeF;Lf|$Hkz8|04?JzU!~u ze6dmXZcKx`QCxS9QQi95nc4zDoyB`=uf}LJR__&O5C6~xHs}P|;OjN^5f`xGLu?Jw zXN7ZE&&4r&?a{$;`tJa^fVPXu7y13E`Qm)26f30d`s-!p6yvHmZ9n+>I8$c#iKS1Z zas9ezGe=<^YVZkMp!A`BhPWawtAUaUQ~9sZL&qnQowegYrPlVY@94;agxp+r z&qt@F-8faS<2Fplgoh8XEf3DhLltS!0zyC+K%0hr{X;-o9dSq8me_0%9!LZ%FQF_6 z^hH#0)OKo87feSr3u6p6wF5ND3U2uD7Nt3LN%79#*VcdyAQAx8ath^EGZgqT$=q{1Z&3Ixe9;5jka|_Is{}=$dy!t zvIS4sO^2x^MJyddKIMgl82(L4w+9O-df=S6_Q>p-IQu=QCk z3fPeKaQ1NCP^D3p&Nup#>i9Bm1s?@ys4`sl@Wt2FAmaG4GPJaqt~OSBQ=qH1$Vb&| zq9TFdd|m;k_HVl6^&JZ(209^G&x7gq15NBlhUc@=>RaKmn=RA31;s8$db)B@GvWseSjtDcb29ewS*l1hTGK z$hSD^;c=`yuAXG5U=W#Vzq@03Z!%?v;Y*`doAjIFj1-~pvF!#Xg3Y**>gW27-y^Oa z!T~f1WQ`yFu4(SD$BfY6(gWU)^9&vr7wNPo`LtxU?dF;C@so`Wl|It-v2?zcoPTNx z!N8uw2(X}{DOl~1xG+LTfa}zuA63ASI*#im1~Fj88Mqbe76*QoJ$pYC6Ky?0w`dhN z%Xj5~$g{!B8eQJH+aA}8*i}0aX|>un_pa6Qd(HTs``mA#;t@P2&rNzV5l*34MBIKe z&IM;iYdG?v)1S3rjF@|$_a7OiWns2^!6HUuKY<7BC_=P4r2#2)++zwp(@q=pXxfoO zDQSA=pYDNOE%Q7cfh#G_J=E57e%!(D^K+cZnUc+V{1ucQJ#Ok&fsQ?X_ad%KIe0g2 zX*7}k2Jv`XER243ZB$JnYh7r;9k>>kI&ZdgD_GjG*334gPhe^A`Puf>t(%40JK^{2 z#=|1$vo8Y?zWAzMx%Ju^zP2%XxUnDsj&tgrO7*+IsgvQHdaOCciK!@@Rtg+vZ(q)3 z1tkhq`qxLI@bz+C8i!7RjQ+IcT^E_n|yIH=}@29 z*Gq_%PCR{*5ywJ3f8dib%5b>8k^#B4eNx=Y-o-qU0VFXR$3VW;`4I}@?@)r6g9CY} zz*6l9^34U4qH6@AV`E`!1Qrd3BQo`^Gi`|^A9bS@vU7LN$rtBPbgur?31XYlv% zm$QE|qp4jw+VzYaqo_%?kq|12blpo|>ArX#^yKkUH;5e){_*=bkKuVEcmRZ=LpYDI z&_bGl0*iW`!e;u=w%C5CE45JC%%w6~dIi45J2N)H?{4;j(#-}j8`Yg|X>CenVX9N8 zc#^w2)vyb{i1N4o16R`yAg_QQl>mDpUtSxFYJBKJDThuZOuEQE zS=uqadn@=hraOqns(JVV$%eidT=PzaM_R)GXu>e_bB1RfXYl{^Ti{peMR8DlWblC{ z$c1kb-TbbshUbl%o7mEj(mU*{Ma0piHBlgkNihAiKh+R|6s&1?2oq)J<4h4D{ zH?tcPx5NT|l3Jh`AT_AZreUZNB%2fl{9sKO3f1CxH#9h^uwr-1R3(YJRin;s?_=NY z3m=_tJ5PP)RI2*ko`7)gzp4j`vC?E)F9iJ%~ChXOHG;7#JqDqZW;Wc;i7!Q+LUCY%I z*tSt_Ccc^J-l=su3MM=LSmqt=L6t1YFCs}rTbL;Ky;{wUO_eeK369A<1$GVZO9W&ZYk8g@2&~=A4VW9q3n9Cr6GGc`~Ats5Z1BOPHF74yjErM1>q5z+Bbj z(F+yrx*lphaRZ2a%+!~neHl!fd%$9Qg|IxaR`g~Jp8T2ePsPg9`(-0C+d(ar7(5HP z;pCC6ca;foZc}&`(&#dzF1iTqh}TNTBk;ZHBwiFw{QV*4yU#{fQ}Sl^Ckl3VrQIlr zvS#|DxuCpvw|YE5rIOkhS3T*l7D(8C#1_8qDRN;eBsNDVQy8={H!WJr?l{JtD$Hv8 z_Kuy3R2|gxSlw5o$2MoCiczSiHh+-5+}k97Tg-<1|Q5q*w|m*H6a$ z^sca2TVJ3+FmGs8%H*6{9BM){RO0oN64qozPEq1t+wa@GqNjcQr8v%&J#4QQ`lq6B zl#B!Lk2*Lmt(eq_EH`LpSGj>cQ^A!f3zerF<$Mt{&)>E7_#SbXuKt!`?T1~f3|Q+V z`MxJRz=|W2llF%M@KOiBvabZ_*XTabsVn2(d09D5w%4EY&EaWyzl~M0qTZ~eSDOE> zFy4cT90JuUe7#vPMsNT(kA21QR2CcsM8cJT;otvYQk zka8Cv_Lx+Vc!$ryT(-$_jt<}48rU5pOgx)2PxBF$I?nPuN!f%WjtREew>F;fB1Ya> zhtFL0>cite2d?7~ymQF--=x&e@sI>CCA7ba?8a4!ZckeO=%wtE8II?BRh4*dG}Lmu zUJ$c4ucXvzr{^mojV^t0un`zXbK+V#CLPUekYDaUXmX z4HVC~oN{x#2Un{fb%3{~3Vjp^Uh4StsdYNSZ>&Fui%-|4G+o(cUa`<^d&G7&INzsf z@WHv>5J_VwfAReRxC6E7oikiozeV_vejU#TfE+YR;wdFGvxMh{!{Vg>^2-zWHl#+- zIm-)}0@0*;SJ}n|is}Q@^R%UuSdsISq#O=_V))=F#>W7<2fgn!6zdL-pe%}L^ z*xwFCIVEcFTCN`S3tRi3uc+^4sA`eiipcc4!#8{2<}q&a7_>YbxdSBXynAHjNiMLh zEB9U4VH^O%?Dj6Ch988rc&J1f?*!XgqrTW*qcG;s*C zrq$AxWSLTZ2ut=^7yGZP>wt9-;zbdF-hy?4l zPuPh4JB=eMh2>_nP}^XZpGSNDn@H`Ob4_tGweL089n~FL-=Xu~a#C51Q!Th*SSslV zfFJpM8BsUYzSTDI?1**5k+X>3p`8Azq@ndhvaSr&4Z0nz-o+ zpQr`1?pcbHv&dR(zFP15Ib3*srzugkIYTvFUt%`zW|m2s+N*EY;pY+bq#`&9Qi~ul z0;Qg#I10k^Y%JoRc408?1`M5%WSoRNiGJlW9Bzt^CbFry z|0)M;m#66n@_brknKcOS(R>Xs7hrENq__IP_!G@YFL~YFk*_eUR90ut`ZyJavG2;T z(vFEX{qDJB$SwVrZ!DIwi`^F9pYTfDAw!^o0JwDfli@zQp=Y34V+CQ&-D~g9>b<;7 zy_0klN@btl3C9D6q;nj54(kRf=l^Ob2W9s&*g zysZr9R*Id~)U%kCZxsD&pLHc;uHa*keIb&^-f2xDP}N5eV<#I7-#)lh(o|Gcdq=c| z9A}ST=>wU^7NlH8brS~p61N(h8<^I*HbSou!=RQdfn+U zoW@A!$UuN+7GxUuEU^Ak$xP7aLHZ2m!h?7Z`)k0gs0qwjPM5oNIZ_vSCv>UVuxMy= zZ5C6BtlVCpS_yMcv36Mc!kmpvNoF2)&)6(vzh*)U#=b&E)}jWib%LZ@6BY9CEE+x@ znH~?ig%f`7)j*_UPYP+c*1TuJci2nwiSmNhV?qoZ0uTD9X5=r)48aGcZY&U)-KJ;2 zhLrS9P@YKWXR!PE9_L@v0U+DZKnT_dIyam?3Pn`wAunp9-%oal#4C-(cIQ(_Y~0S< z=F2aWGN#dtKcGYV%Ed&^HVFCAyi8E}^xVj8By256D>5|0!03@QBh3-edvVok32V{k znF3w=N9IN337b@NsaJWc`~2KyBEbMnA#s+mqK~OWeL$k91OvTJ1d!+pDZIASC!80c^YRiS3^FWR zXxTE}@GyM(b4qS9@PAIpv66c)UDAui~sx_5jG29@C`gP~G`(W(rW8lyth zLX(v1gFEs8h?(-Sd^=kV!gjv5u0Fc~`0B6Y?Y*fJyqwk%=Mdk{tvQq+2gpG6axg~U z2)s_z!^@lGz=hCl)Ymy*Wu!pWE|`JX^W$ke8>sFFW)_UAB8c#hzkGw&O;)BW{IT$c;oev*OJ)Q*P{ z=7*%2(UN>+e*Q!`m~LAvNA?T^NlJukMMiNOVe|A4Qbk7_T$-h!F!FgOp{Nd?Rx10* zo%m-s)FK(7tAwb`I?nX3K072f2sm#Xe?ZG&bFLLLbeo%LaxHVSqVHpXrboiD)AJ9^ zgjNcwh*rXlVxp6S_YiZ-@6Fs|y^3pb7MP^dp@`meDq=cgFsQM8kXHU!HLD znZ8z6ctSK$WEfMxHpPG}U_ul2d6_%D&Td=&!H45sdY<%F{<@uZQ&dO zIsQUyO10@eF!leXE{Gg@{Q1n!G~8udEf%IwHMh$i3)H%vNt9sX&$oow0abbt*d$ z=&?{ODhf76>r9t`ub~LM|dCgq3MCgmkaPLbZabr>ns!`Fr)w6jAlW@s~2V97dq`^BIumkH8VN z=0TUgTh2f`LmX_vYrp(?X@5Vy-WMwimyKinxohY8w-ut?bF2}zIx2->F$Wih?E6OM z23KqzOT(`f5N!ApO^S?jou6`DGFWhmIP#J$!ZxhYDWdk34Ri#ap5O4(R<&ENb~HMr zwUU7`yIak2E|>L++I5zu29$K0x!gx3Uo;DMOtYQboRAJZqZGHncAcuZX`FQVE-vSc zs|(Q2;?|uQ%hmmN|6%VXQK#xAw}R}pcC=rlE;{?hywknY4z}PE+%z^}VcOf`U2d~W zyB06mkYFlqaw(%t{XI%-V*eo4Zpx5W3#g1b7kHp<#UbM+Nx_XL8^eNUb52cy!l5HB zn;1VeQ11yeN*tF>RFI)GJCqIu_C8M*FQhja+pqYfZ*BKoq5D9U>DZ6rLSCI2gE%#v z4~E9@P76e4ceG75sss-V#fkZ2hvLNHB%y0tE=AD@Z|1%rN#h0ZaMg1OOLTg#e-~m{ zj>rhkSt+!8pGo3%7MCJ>SaWi+8$|H$a9ZiIS|bIe9)9~vFJ=q+ z7ya+$tXwBtu`{{Vm%Tu;t5CurGlQ%ZlmA3h-A@7X+{`el!U>t9cmy@LIE5o~MsN5p zqlV$T5i*|+kMQB3)Ht;?9Dh=CvFp`A&Wytl1$=({p$v+#`J2UBRf2qL$rWRBO>4;y z3{z~U>%2h+gaz@K(@f|`pi)Bo>=jakW4{UD@e??g{QW^$HQ=3i1lBmO9A10x8CLK9 z@KnzgET|;3k!#4&wn>B5uYfYz_T4d`0(nFDv|;T%)1A(_!g)W;hlVRHP1JpmI2v5$ z0?_zK9vc@i8=c*1W8jdfTBPalYXbjl?E((duDU{6ehjCb!oAF|4j%xsg3p3IoLs8VZS5I4#(&u=&|ggu}L4hx7%9x%%3OmNa=A?}7=e(n0vdbbE%v!u)| z_5Bk7(uol?5 zCGhE*wY`ejGOPVj;VkXjv#+D9o;5M0m1KxOXBo+8d?!kU%w$yZ;+qREwW#tb4j2u_ zoi31uf-@-LXFo!G@=#Fw0NZdta1!itSrqFDBB4{r76PTeY4vGoZ$z2Cu>Xj-^mGf=p{ic#XM6Y>Tl@8lXn0pbnEt{@mj@Yh;V zp1$S{A8YHKE4Th4v zKILgdQxk!~>ofS8TrNA2urZg7d6Zd! z{#IhY<7oKX)TWpbx)XFMh8<74EYw^z6Ep7Zy)}Z+b(=O+^rrzg}yvisty!Lj&jvw#+37N#}5x>n~iuBaniNI>nD`1`>oT?EUHmu)@E6PUG#8 z(_QK!o+q_FIn#VYzU?Ea4`qsF%kQHm(cU70YbY-cH5NU zZ6FnCfD=D{l&@dAdIIW$!krBV&<0Ri7JUvC8V;2Uj}wSpZyO$_ZI9r#ICMrwi3iuS z9J2JC_63#f3M`W;<}P%UcWl}HcT=Q2O3)+o%cf=f9lBz(r{#kcQ*|o5rG=Z^r6!A* z58M*b6MI5jL)F-RcjLI{praZeF(wPeCwRJ)6QJcwl8GxCfroj9Qyfng4*eQXVHc%; zjx(Qo91mG9fc}PaxB}KeCFkMFJlKxtGQN;7v*Vi9;`AI@)~WK30a+tmuPRz+)nZm+ zXISYRV#8W@^DHY$L=e|@odejAsB%5VCj~D9>?goiMn> zAb1__SYAis=&yn_#T?KmU0mR^n+JWA=$9n1@iu8LfDq5mHsvu}-eD!fP76N$B^3(P zZ|A!=9lZsE*I2F>&d;;h4J0|tCG^Wr@C#zt!B7~v#(t|*paVVPw&|2R`%)`va7lZX z1rYYJuTJW-JXRjRPJsUAPgyX@XOe?r9yv}HV@J7o{#u5os2;E+{KS^Id$AN6Z!6Pu ztIF?D+ZR8nWlCB1HCPm)42aB}ILF$PC-0`zHKF3v%=X&f0$5WRef!x#;jf0zx_1T} z^4GZHqNuF10HE*-u%9HN==cqommy=inu6nQvd;U-22ou9JWK#@~(m;LMH_0EdH=tOE|G{J_S-btA4oA9JIEG{^RwKw732 zicvRnfO0Si)lyd1i3xM4OHQvsR80+WsO9ob7!2F6s6i)4`C3%fYje}y z_K;cT-h;|}X3ZKu2T4rooIz!;$%V!+Wz%dR>Q;MlMbyT!Z{p2(E{8WLpMdVER&+FF z>kf5{f-ro^N_vG5H|KMaJP4&Y{w2pHNV@8=KQ&(?vVb924xgXhe~)GvYiwRM1dKA_ z4|>n-P=d%s16XK7AYnC)p#77<607vAtdqQq!H_2%gG1BFM_fMt4gIHHAiI z#}&3WiJ#(P(X?(L%Mv?Rahpj7-aOdq+9J)Z_Fy{`p!=0-Cw0eD+l;@(Eb0y^CFlx; zz7YvE9XqY+y1RVo0uuK1T)SF*PS;gIO~+{>o`o3dDSRTC`ta!Q{~0nLQY*a1BSuH{ zzKTgt5iK}c#B^7!c;0Yd|1j*!G)&H&Vp`?uR26ExwP_Vj$;hd^=`&*u@eWW{flujS z`&k6n>e3G#GzgAUi6CC~B_F6|MY3+iy0*pO9<>GkP#^aW!=scZJ76%TUAoe8^7(3X zDCC8Mrs}EP1po_q0nH1u@3i=>as4%;z^sg;_F@h`OQ2!tF&GoOXOy81vVwnB3d{j? z1A@`S>rstIX1v3;?Dze{*nn1i*6F8X2oh%(E&}1(@bA$l-KX=>4G=!)-=XLfawn9#@0qnf5FQy zW_lE2$VMWAVyI6ebA0SM-|C0nnH@B|v(w^q$m@9 z$_oUwm2&Wt1A*hV|ECoGOpGi)0Fz@r;V1r@rUA;}gl%oU*RM!B9o_$73ccLX-&-W! zT8|of^Ol9BBTsUzaDOysW_HG|dkT;+&?>C_JwdC$`UKDf3HcBk6KO_^UBPJ3-fZ*y z&%_KMwy>aMV58O0WW`Z)eDGb^U+9i~%Dv~g>Ulv^U6rP1Q9#=?g0`cUS7rABXdJ51 zPb_rZmDNC}RjDn!*5GEOu4VbmM1xW=fkSG zz`MdMm{DLl?HMr`&z!2{H2X)`+}PS|sXpszW!d@8ja|w7>%MeL@Y9 zDS5k&=v&;}{35MjTI>~P@Cyrz5P#LTB*^t0AKmZE3N1-&5hvzGencY((NA`MSLz=? z7aB1P!z`g7j2|3z!(UTVklqagHD7Mw>UnQq*p60@abA5M9x0(V+gzfw6Wp>i-QH}W zU1}sP!za`qJNHag#~7e5te-+2v>UrC!gy?SS7Ruq3@Z(^{)BlCQo*={*u*x}dw6(O z=#aGOS0N>j4&bQ}dca#G&E7KCJvJ3G_rp(r_I~oxF6d2Lx7X^i2yGnmJ)Nla#H#zL zG+egzapf_3qFZoW()$6$wvXllEVauHj+NBp8^GAu{|p2@CkL0Ue7vy*_zl;n>!;uLDqJk*%D>32UJ}|NS&Ahh?b(RcWGz=~)vztT9X_u))4&7LeQeZ!Y zo%=^)Dxa4hL7q?)+;S1$DF=AnVrio@8S8YY^@>>z=(d%+E4UX21jInMnckfEb%D*V z_j|H{_r}pHpIGXC>on9aR4CKjrE>El*Ver_k-jwQk%jKUoO+&J+f^%%3(2GtV*aua z!*5;6P)_WJ5_yUYU-$MPTKP)LIqvkmKYd?cSw-;zA45la6l|DBj|_a8{?|((i1qa3 z1%yCoxG`UFy;E!ElP`tyP1S`{BA01mhz46b}kMsO5g>mWle^K=;d5SjC1w= zH(_7QJQY+`YlFIfE!)o;F8!nVcTLC36(GrEVqGEh>+(Ro+@2dg{9C!b?h`+tWp*Dx zT_1CPDwhJj$CoU7C1c&Pe2~U=aYk-vTL+OnpqR{ks|8(9CByyN*$U9Z)w>y^9=ePH zqHx*%fMMj_pW?RGiQL&U_&Kit-WUMhXfdeL4Q7x(kb(0qWqVZf*N%ZGKyc0MYLouO zVYYZ}wBOa8D?Vt=a`w}}0dz^WE;yOqo1XK&tj!_B~X z4HDw!ur`qH4OE4h8XS6_=$>X(w+@Xi*_Iz7zs9oQ&0LilxSqr#E!gJt?&P8PU)7T_ zsCv><-8ad>4!=$R5(w{3iHQs#xF8z>domK6cln>AYX{KqwYkz-8KDzF*F&b%Snw%N zgNeeXYP;{9pzWBT6|fPi?$CjUNG%GWsAG>HW;PzQxvX^u5QYrqI4Hv>NePBCs@QSq zt3ZFzbeW#wyT#%s{TEenZ7fRZ?ocIV!@c`|8O%x<-=1L*E#5EAa45&kW<@W(8Gde% z5OSZ012Hs7B?vUI|I@ond~h6aa2(^*U)TRRjyGrHg*dkDqeXI*>nKsZ8hy zi412&Z8^-?XeRB@QDXi+jW;(J{bElbStHvJcnPjLP+)RC8d13~bRJM{5RV-9!kZ`` zm+e3~{;HeIL3I;30jGdn1QD71?wrylN|9J5$1R+>{$(A~a%$ojSNGb|ePW zA_<8fD#Ki{H$RPZ@U7DmR^yLV_kc`?GO$2thE}&TqVx=4Hj4N#Bkn%#y1~@VDt-&? z$5Vu^kET?Hx-45EH>o`RW7X7-sj6@~@su{kNQ-a&tI7Eh_)Sn%dHTplUr*CILY}cU zmwlY;L>bTe#fFKtd-}+lu>AHFMufg34K&0EZf$#5^6bND5-IYaS9R2h_x71(*T+4F54G7H^ zLa<^(8PL*t=)6U%^}Luvj$OZJYc^007q+rQ($cCXC9ZHeNM!GpDDllNc5e)PJVRq- zp%_BK)*^?s9HWD5znGIqR@Jt+B?+vzk>lcLKiF~Oz}^M~rufr((8K>RMjDFF{x`5q6iwgf%g_m!o9!%4?%@_Y+~zpjaf3HUBkk1%v(CKh8+qT?U5t=M z)s7MV&u0j0o4jky8EP-~v>JkX1fVouZ@dA;2vHNlqbzuo8fY8UyLh<;dyPZ?EPwuN zZkAB)UO)Hx9#VUEqW};|<(UdcWSRb&yxnZQ@X+pHQ8=TUxh0r5$cE722!syaYwdUO ztW8F<$!{=ewn!OojOXQ^SZ2eXDH%af&H+v4{*)Z||Gc!Yd;HI(g`Fh9TUzZ^{Nd++ z_DhOMHW(y0S5CbVepb2Ifw{5Kf+A3v^z@7aN^{U@2vW1Tiu6iEx#6pEFYe;CHKyh= zQxiDTTErHGU&0)bAHlQ*?^=do0^jeS=1oxGFHweFD#GgSvkfjT;Qoza=P+j; z%@%2E_1UXV4j#Y=UI$7qS%1Edbp)S@KT0|DEk>d}P*kg!SD~K@dNd~oVoMb?c5n4NC)1b==t?s}Z7?3>D^i3Ws!=~_4BC!gz# zu8fsgJBRGNiIqrV*%>jNbt*s#`g#%8YE9MK9rBrXZ_~e-@S3R|{?=D0+kb19p)kZQ z;|fxN?`=%FS?q99tHryFJi);-iEn-hp}s_nh;dN6XCSSws?8@lyyOD>mRfi$ba~=P zyPS8~6}=>I(*a9yC}^1y?FB^RmBp9vr^*C_OiLzUT7vDXS;LI{ytEr~WU3+Mpt*!; z;nw^|w^(Mf&1(dV$55M2I%E?De}qc_`pK1?zUL7|DRX{MA28d5Qrcx#jQ?ZdndsG` z>32xc*+UaofTjr+d%pwW33>~!VT8{o8$le2-lU!I5^6|Fwz1v>4Fn!7h?23)@Aaw%Wp*z4Eh;y?h%P$Q-El^m3u|%CrTZaouTA#r_FBD4KdqfIc5Ly1 z`qcd`Zjhk>R{?57IDQS3ww8o9pXE4MT6Qfk-xVBGcj#kM(;*ef2;=A&#a z2SC-lP5?#lB?KKiSN$Ca_9)P2ld31xH(r!e^1I;WjwtQrQNX$|Dxdp^J$==VEGc4H z?v1v0*RT!(mn(T7hNpx&LQ*f1^&_}vE-O-dS6-z=EOFE@(%vBX?Gh%zK!Tk*-bl%b zezp!z$tGe+z6-Pwm|#(U6QRoq3JV%%nh?fltx-JyXOXIk*}%ra;&B$)d~LE?WptWJ<_&&oI5zMyG!H{UF3eI?v@ z-F$)*P&Ps2jCLaL?y}0wyd~82CH^aOFFp1tOw11gCRtNoM_e>PjU{-s)2T1?=aq zWW0VY+Rx&AEIU#SCUKRBmF%~1SL@-C#nSQ^cS=)n?@T#6{AYh~BaHb#MY42yd}sGD zM$wM863%oxDwdXff=NRl%0dioLp(%!4)hWkr;WG2#fZ@(*GGFS>dzNNhrVS*@?24B z0;C%U9j~8*ke_*;%M9qz7!Vs(mADM4myXvZ5>LgVp3TiYA%T`R>r6z+Zo^B3pWQYu z4NPJ4uaLH44>TnEiHi27FJ-4l{9CK4Zj#Vrg%T5s_A(3U-!4exw0$n3NEgzM>Y?mY zrj36h{_T3s@-bUZ+?*(@!e?oP+?vOU#D*m(9gPi5|lE6$9XM_rt7)r2t7QP{*fQ&Z#<)6nZfz%FYz#8-Mw%U~=H)M}>rI?E z)Hp0UP52ysTI)f+5?PCRr%z(hbi+gTk#eYO^Si2>_3|2Flp>5ZyrqQwXpD3f4BWo_ zhK>Z}4dy-+d{)_==ZfQcTbhF&5@_oSjDazwDZQFkKTDu=Y!25*g(`)ZF zm0tn@#wf;4s_dlq5N8)yZKH1vc2$JuuWLjA}8pA|xsM8xr_C_6J-DU$4LhmyU= z*)v5&C?T8dc{n5Ulp@Z`I{Qf08E2n)e($5tr_bm6`;O1|`;RWg`}Kakp5yU+JjY8) zXR0=308-orhJ4trlG7c?cMGqo$d@K?5A@H{T@BplLz$WY?NI^n8&MAy{eWW{*k?cr zx*FX~F1!tB{5?-r;{}2G>E9tVNllgk#xv^H#WO%X$+@o$6GT=x=}!W?27`LUxvzZl zvD*Sb%kg`^UNcM@c|_U)ng9BET}>KJfG(TNUMi!`y0%(T6v2e222kB5hyGr<7hN{$}VvMmD!05wJftlwqXViWl6)^1Tg?!meu!MO);hPw6f60^+=C001Pv-ph z{0BRKu#V#HLId3m%<#X?idfs8pM+|hPP`mgj~2kC%4-K0`Olp_S!G5$9$TN%ed%TF(=JVB=uctE zevjOCsn%1Cj)3dZH3xUU~>L27GMsQRSiL2&Xmy}ygiW@N}4%~NjX->zeXb7 z(FIQANp!EJe<8{f z%k!BxdU@eipRI?D*7A|==zBh&`)jFJP5f+M)s8xSf4P0Ng>N(`m;CPTOI@{6d+&l+ z)dctL=f>+m!`d1Aixc%_n9F{N;GYH10i))C8=5V2fP?{oWak(a%=w+vKGqVm z3V9!Ivrk!DuU-c}6wnDwM;DHz>>G^hNbV11+wM=Q>uj}2w3M{wtOLV@U+HqUQ*T-|cmBDF!S{f{%MqK~a4*vSi?2R{e$54-6TA!T_J17r3#{%l|LRH-Zcz%c z7hE;4G)A*7ubrHF{kbByIvlzdT$nv}+$8I<$X8IkB++_R!BF;pw$Yq(>My<%jEf7ksRSxdE(XJHC68n0ly3v08~ExWuPViAOLr!R&rB7 z^#FR~uim33HKNO>fm=I|>iH1|uCei4_;~jNLZiTkD?I}P*?%Q9k!OB1;U`0AMWC4e zYisa-&glUKmr}Z?^kYqGl~~Y*uryXXl+D*M2ICuk$&EKN)l?YqDUQa6YD2oQ(8gb; zQEVtqEBr(QfaqjRAmWz;`~Nk7kb%Dt)jA!rwmej(Fx~Q6+rgnYrM-wSnvLmBdU3I8 z7%h0P$N+u{dYpK@_wY|Z{tLN1V7NKBT`yLDfY?OhQv2K>FMCaSIS<>iybUF>Jhih= zM}2eDfPy_p1i_u1CB*ArJJ5O=@IT_8x9{#Z{GuHw=CTn8dd1(=+dQFlR_F=Q>dxBq zNG-H5M>Q$uD6PRaI3}}bOVLqpH6ZHF+K5YeoDF-O~HJZlAW}l zUhqntE7En+%l(G2CneFa&VxJh)KYAMy9jBzXIHtyzVR76YW_g^f)u}YMT!3p!RnO_ zpMpI@$N&V=x8)8U;A-@v>C=wSo#=+8mgM8Rw1F{8PdMed0rpmi!^{#t+UQ@G>boMr z>bLr71d4*NuvNb%N&=a2-j53T9SRXNz4rd(OBxhr<)|Kl4&1uiU^wqG640{)*{bo6 zLMemszRQ)1b&HW_#JZ};WOdWBm0I|npRvr9*JICiz=e=^HLe22hsvu^D2f z8e%G?1Zh40Q!Rg);gs_`-KyyCM}D^1j)%<`9C*66tFa%~5^)om&>vy(*6peL&u!VJ zf;AotGPVONFw4#6r>xMIgFYVTfL`Qr`uW<*yZ(WSYGRjFeT3iX(%%I=^5Mi{>sLrY zMoZ@}5CrRh)Sh~X6qGOX532(rS|kqiQMO~-4aINOq*-@PYIx(gmddJU*#y!*HaJtkG+EJqHTVKgZg^#iM*Lu&(TL{B>n}LBAmiFopa0g57Iw4C>bF78 z)U00$fLE4RZfXfq1x5Q1IQu{CGVrPN>gp(ZEWa-Gz-oMNKX)_PCuLl1X<=Z&7n~n+ zM#2=FygBtvl|z~?fq#$ze#!bq>9Kf}rE(66t3eF8Xe*m}qvCDymHWO&#AmdpJpcW> zM^08yO2hhrJ6qDV@+S3<`|xb9yl)aE=OgdCvvE$vh9fe@}Xh@b=MTaFfC{}4lYAF}n!dtWor z{Y}Dh{d|Ewz>T(H%k_1$%bp+9B<){%fDyjDt?FLv7Y1)Qi4c|$PgsJ`tTu-xOFBRO z05C}I$uBW{0$f|Vd$;iT96nIk_ph4^M3;P2r=l%34V>O;#XJ?wM$z_v9*wiZd%+2WCHG;TGiuRYico|`DJ;8I)XtyV4UmQqEg@i#~->7lDN zWR0l43tqUDzE5iZQ<(zfVG8L1Xrg=R$8TpwES1Xn*9IKFCWLbpg9U$ws2dlK2H=Ec zQeh(a2+sfwH@OOVr4`)5P}GZ6`K2#!Jfdgh!--T>Am^-KKE%I-ysUj-iTd?k?T`QV zUWN>Lq>HmVq@a{l^z8H6tvv~$H}wvl0=+ik1>O{;O!y?ji5wOo<7FBm$tgr<>@ovN zTa}`8s3kuZl)-ZU3h9uo6WkAXdgxmWaA()VUOLbi5Y2upp-rUFo2q|nrX!fp2a{`o z2sM(VMp!JYKI_&@@*ube0apt=;&T5n6>r-+`uSit(DzK@s=V)|(S}%Fa>EFVV>?yX z9ns|&Rl8Ia*fRl_Hp$86R7DG!aBduY06-rA0Qe}pEawZUKo_iL3;i2pBlHlhKZFWM z$x|%qAIa^85sybt^{-P0;xiOH$2%M0WL0KOA1K^k2ECS6Ua3YLI%KFwTN8?yqOF4) zwnumdx6gtiV&h-`j>-uS$){)XaR^`W`3ZI(b+g+Ff79+B=!U*=mkTUn3f`YQh`b~` z6&zVAK5i37!gq>`52~D=ZIkagK-_O3RL66oTMsY%O}FT@($C|Ch1XLKq~{v;GdNCBNewv)cuusj<9f_yE1Ad=us&(t0mtE*{vy+if%S- zsS)eXB&;*EW8Q>CUnMO1YQmgezpiGy=FAU>z^`RfOk32?Q5qfl_0nr}bYG}W4~hCy zJ8P77OHG@OM+AK}FY%@>t=OyYtEk7UA+BCzBe5@29MWke14P@4)VZ}T;S^v!nNMtY zpT_k|f9K5sMO+jx%J+#oF7>@v0cb(fp3<&QB)=eN-rLVv z160a%-W`l(xdgJ8rGNBcz~&M$ALX(1+UKk26B0F48?h(eRAYUm0vagDWp*1OeajEQe(*| zenYJ|VDa^O)RAqx=lmv#Y%V>m(TB#=#xI|5_-Y8%aUz8c6WEkhAVQfs{{s#%(@g&g zVvhn<4T&$E4*meB4JDC$JE4HwI^@Jwu}HpB{mT!W*ec%A<*(!-t2>p3KxWvds%QFA z+fK6kY?cCaog-XyxdFWRD)TY{j}Xwnzr!PDn(XW#rp*`Qvr!Y{c{8mh?6|XyF$HJ7 zZ20tKK9zwv)#LRcrQR8i4FQq1RToxk?@D+)4-zF}@jeImtJ)Y=fWA*8J4O$E#o{o-pb)&U0e>AO8r_dxYyS|5G8l5A&eN>?)d7!mCg0RzDJIi*@($fMEKc6fA zQc=!-@%5{TZ193#n5cT-HlbtNEmkax72!+ssa%nAGL*ke5|=-k8mQ63BDhI=>C|&- zzS=S#Qr;7TS}~Nd1OWJ69y9YMPSE>_y{T{i&~GAG6VGT`PZS%ZfB~`pT#SKRPR8L! z+TuIJO+Df5q|r+2XXI~(p#a`_PdMju*2=TH43#?l77u|2O5};BW%%rM5$pYV@L0lm zPm_{?bg0*5WRBev=L%|b4Uugr^$0N_St?DH&N5^qQzv$F-q9&$*4QQO0vmTe(6`>f zrf-Z3YUd5=DhjgWUM|o3T0%Gf;#f$UBHj6lZCfeD&!^^$|GfM~laTcY%zUB*>9vtQ3+jL+)cqN_~1+#{~V*aDX1q1AcUaSARlb zJTs(I{r;mby;{$Xcy%f2?gJ%6(#p?3lZxY;!7)$>MY;h>Hfx5<{gY~&gI$I5?5<&* zb!@5xmvv$v9l5>R8`ec=U$;l!Q*~A*2N;V}FS%NyTDYD}CK<%_+xIi?D(OluAD@|Z zuD~!AvGVz$GIOPKNq7Hdo=2_-9^|4IQn_G*%L2uH)0wdgc-h_UhRuaTMEd24!23zt z?;<)c1-5OpS>IzZbm^YipC6C_TQ|o^B3_Q`$@T3jx|fpKbBVE{WU*n%~cfxru z%!1KoNFYL!#}uD3q1AO`s4!R0YI$FLZ)qytW}R(6yZ8BU$lKv!gVgWs$C`ez_^o%q zT?Yo}MIHW$t@0C({#?+c#W%GSN)V9Xm2-k{QvAWUALf2_g$orIf=lQC1b9&ay7BMDoMLMdxTVjxn+5rSm%-DKC$K3AhRU)VQOLr`eR38Z{%6V@ei4As%?s! z3n&@|BXoJHFO>8_X{_O)E5}@P~DJTux6K@Im0PW-Ng%r*%Hm%=q z97v>6AKaD0r}LZcJ9LgKjqDWD&i}Lsue3cp>+=J{t-`2zJZ!lbifqgt#Tw=ENp)1sE&QOGwHt9Xs$4O3>{a!o zwQk;^7nymc4IM`F^+e#jMsOn+iW?lB>)u2v({Cr`Ppio)g{vp!V8SlP<8Z*;!&9au z{i>cxhpIjAGUu#60Y9~L_)3KlJZ{twf1+ea%`%C@@jPoyK$@`rN; z2J00Hx4&Hux@bpdU3lWZjOrp@{}XGkQTp4#?dqyOrVuKM^vr+ol|9@ zI~Hrz9KpUoINhZ7<{bdG7v2y8%D5+~rkX}~A&Rl#o((7AoHj>9{8}oP{qGwx)BgYh z%Ew%wYc`81roA`fOZqyT#$Zl?9_4QDi~{R5T8%&I!b@sRk#Y--v!_70c6${xBeFJS z7Xp21Jx0sacuF7{TZzRpaj|)gr6NhDxsi zK10D`CSu-GplbD`lI0l7g%8>})~jtOk(s4o$qlZssy2CmzGu%zJkw~1+y>A@EIO0ZW+4mp+r}Q zk?u%J?TS&^KImlDn$NP8DN5P29^%9*V;Z=hpO9Fev^f&KSTUQX*YDUoJf*tV)P}NI zE3m2vn!72y!Y*ieVvufl(zBezz<);MMHKa$Ke{(Vlo0t0qhj+%0w3XrTQgb0Tx?l= zXX@n1vBJpvQg3VN363S<-#eCL@4^6piMKaMK}qfdYpD4Q3;0ZziWS~rpouZ+`&GH5 zY`Z}|)Y5lRL$A$UpU(mm)ejOUy&+!>NROtG)$_rKBo3}Y+~#Tjl`lM^UR~UW&fi6F zl|LE>c_kI+hc_0;RWt-mFVr4B-7EijH{gHs`iaA%E07(Tku z87?xlWVG2t_SNGiQLDiGdll2ZyPoKoQ(K#fgZ|6Yd%dh-^NHkfb{+#AK_OA5x!ozQ zF`WDjrfB>_=&EL7qjjvKL@q`@)g&WEtzOH}HGV}hv;Pdxl<72(sM3vss(1CeV%KDR z>l#bMs*H~dYWj{N4i;ok~5H~i67GEKCZ*gHzwse$@wH$u2@=j zN@;NkNPVF>A7@48j$-ngzwd92Kiz4%#shKxxpN=1nylkv`NpcfF8w0Nf23^s$SChJ zqE?qGU^U_b*l?h}71l?tR+BH3Y2uQk&T9yaj9gPItJWde03@K+7zca5$aD&fQ-t6E zD50%$Q*)gPBqj;Dc#BNq1mq;Ramo4|N1%)Pbw2*Oicf{MFs5tDl` zybo>R2;Z8@PqRS=^j4fD$ZY5TQDzzK285~lF@5GLQN(ea*4yRa#l&eIH_}7}&k$$) zs7JG>y7gKVTkq2l59`hkm!Wtd*Eo!`vU^qSYGRxV6EMhliT-F>{a)z|BZ)UGwh?9` zD+tn0E-vLZp0BzEh3^f>XjLnnzi-k84eXCMWHrDz=6@M;iSL#T-VmXpVeS1|7$^b;2kpK|+JTi@y*=?NO+(p#h_FZ}C+imwiN@cj60E09B zJq$hu+1v!uagZd)$3>nhs_~AUsCGQ;u*xb9@tDrBPqg+ZbQlV!inBx;)nhL5z1OAw zBwF}hYrdFiy|0qiW|uORr8F3bJ=q^ifaX)6J@k1zm6a|V+(gZjljdtT?bz}!(SiKD zGj;uz3v1U?i@%Pm^}LI^l%2q(HfX0((gz}HvK;u_ef9#Rl#XU4i!HoOXp(Y#y&yOm$f{?L2f+gUHy9lU0DF_YOKv}q z2)mTIQEie{w~`7YNz#d&0j0l+B2C40M`6klYh56H^iUZ&NZjUY8`nbn>%Pj#P$U$i z#A)~0aeADUrUVrW-g~M`2eQ|_dT=TD+X=h)rY?-(chpTlgf}RqcRPv=cXyzMyBL+3 z<$4}eCVav-OYLxe#s|l?|Jr8Xz$gKr-+iV35A^#NTTl_xX5eWZknPO=^|h8f%Ri6F{lUpveuAz0XA| z72e;uW!xiix7v(E1&^qhndD64$3?m+1tKLtJDaCd+<(N5xOZ{Oz`wx}kBtWDK~MZ{ zv5TDu`A}^X&2KbE2&uOS5q%^SxN>axD7!T@3@ImP880JxlrFsaQbFJj-v`iC@C)Y~ zus8sQWT+lb)_&LSEm-e?>?omi5#ItGvH~JHdN;pYtVZVnkU_eV?;iUduK( zdIcw%5LQ7+mm0&-(>^t~Zrbd9(5anCw&Cu2#6@rgn}93W|Ddz?Xl?7$Yw-m)E>o@) zw{S-T5rrA=ZAIXOuUm@M*BCb9T{}m#w@HFVGDO28`*(32y)(4DsIebw3Fc!iXw+64 zY^=o7#2N58qbYTQTA)(NCB4(TDccT*7({2$#|6>i!;#j?)kxOf;p%xk#t!GeS??!n z?+asT*rQ!eHY$SgV%YD331FXeNdq>>i=HED@C;}1{oclOWNA&P7E&&T@* z$eSiYUk=791ZT64k*dCDSC!ej)G77o)#jpT?O_^uMZo&wV#6BwK9B3mb{Z^*EwYq) z`w&hrLO)zi20~9>ZQWUn?rjpt9!8b)igKMdvi3bC-}i+PHGj~Z3KLDts5yeBT4Lo) zM#117D00wtv0}vBh7Y(LlTKPOURu~Fo-d_vVP72;9D~=VzNFUEAz)hMt)%v@%Bgom z3QQP6Oed8VLt0w?W@7-QV-Bth!0QwrbRPA0 zlcTv2_%~~H^F?w;$+8>Yk>71I0ofCKT;-xo$f>j(LtwGg^OQs9p$S21t2VcPB~^4= zkN`5SAS5G?E!e$Gz{IpM5eIR6i}EC~a*~L`LkbsER(ff>ovrG54o5c|x)-vtSwsqf zTVh=49RD7l$o7fizU(Fjw(zQ64?E%>9a=+z6`6;s~IM6JDsAPr`}Jr9WMr|ODXaVe20>`JeKsjP;NdHCR% zSGHrVZ%WltG^ct}VU1Ncs@|N-F2>t$eVoc2NfIvF{D9~lNbn93oPv1j#FsbY$W{KC z33)_X)woGjY>Rd*9HDO*l42OJUwVg6dxQ#LJfos4j;Z zL3PXAKkJXsKy4^%zmESR?D?(g?aNjTe%}quUVg$>nEjlltbFWD$Dg)Uj{LJ0pdUjB z?J3;voSf)o2$Dc2{<-n;?9YjD_;owOwN2WJ?M}J4sFn1_fE0-t#C(5dTt33e_;i${ zMA#iKb_svY@wWs5Ug3Y2DC7X$`$`C;9OzD>Uj?NyO48bgX04H|kP+9!zvdNQQJRd^ zOZ(o`64+BLB{rY!S29*w;}lpedo*V{7&d`h_$eYb!zY11xA8v0{h)4c-G394Oq=z? z%)|8X0e9qO%&lKEsz5%{JtDr}qW{fx@5OA3B9Zcbi*-!+1;Qc>=zyyzZ(leQV-93W zqkvr_(3XS?oBk`>Qht^+plU6xV!bU1idzi42HtOMXpnSE%iMg9z$-0RZAb}Q^)8TD zZl|l^f{yTT>CD)(|B68c7DK^)oJsvpH|TFl_w88|i<5m{J2|7x43!)z&M(8SgR9>E zX_xelisYWa1^4-Qv2Z9R4v9Cy6);-|AWU z1MSp9YdWqwvHT@j(-go?E(x)cet%@v)bk`1xC9zbo?T_VR7&g%2! z2N?>kzR%;1qw%2&-=fYe_#Rcs^Q0LS<|-<-*U8z=Romf-a&5Pj_~GMMwo!M27LiZl z?hV5Dj&lP48sY%5!2SQrg8UU=8H$hLv}tj*pTm(`w=*Q2oN40M$=|mT>KL{Cc5`3Pf}L@1Eu~y3B-#lO`YTS>~(e9*Tc@} z1`Qui$C3YoY(?KQRoBs^SSjoja#^Y;NR!@B zC?3M6`KZ*m)^&Nc?u#6ihP$u6qrY4o%dkx4>~DjX?qGulI3y4v!gDzzbygQKLJ9t{ zzlg5zHKBw##^S%q+j_4Zr{x7-$2Q*p=9of{L4o|@c>5XPg4b)n-u;Xt21{P4b~3cN z2$rn)@U@)c3y;P(Mg_awM>DEkos{_uh&WT0>*OfKQ1TS#M0#Vlrw*KXvv^NgIYa7( zSQ8dLM`V#4CUm5uR_$Z)j{n4g=Y)T;V8K}~x_D2#lv9AN^U>%b!k`4>YgJRPxSrXb zw3@OZ>?WPbtn&#n=-r@kfG}w1HFXYH;TQ0P%=vEACd0fp{MQhjsIzmTH82S|BtyI~ zdM+8&**(4IDQ}uKi}fw)lHN1z9~I^=8801Y0!}mHtS5C+*Lr34BOsmY>L3vs!ya6y z6;HWzUinwXtDH$7#yj9U_n*-L`|Fc(i9UUaOTd*s`X~9UiizSSIwI0n#A9Qsj@MTK z9ox@|ja$o5;4$PKebDsmi=yI$R-<6io3r$4EWI!2&4h)^Xe8lz62LlHzR!P)rq^Y; z9svF9$Z0&=@!4`v_>-`whnRS&Wq>8IlTJpZ7JFyMtnTUZ*bp1bVN7%)W#&u@O0hk- z>Ty%2K`UwZm__oW!Sye*&rqkCZ%o~^vXr0~;o-;D|M6P#mKZ#l<-Tx6<(u0JTUr{O zG&wa>V&ZvXi9Dghr&n+Xc4_jkEE9(Qm>3@_iP%Oga%K#boR);@JfgxZTtD=*8D`sx zi(mfk>oEe^$W6(-EW7h{Tl6g-lwcW7^06>J8a2R=*&%bXCv>PZ;@g%=e$jIV{|6Jb z)5B5Y-*+Dg{Fxlba?>ru&XK*7bo4=br?g4`v*A}!G<>H}9YT&l#W(j8>Frs*zYs1Q zeW9orHXfb4LD8$}X<#KLwt>z@jSUZvr6+nfFa`0DO(kC(QA~+8^iI$kD!#dxxLX4+ zp@h%860Xn*8#!$$ChBP^CQP5iVp#ZW=svtOVJiicEUb_uMUq5W`ue<;B~t2$kKwaR z`9qRmkFdc#AD^+HB>E@9rYh(pm<;BIoHz{Ww+xLFg>7wZ)*vO1u+VrOzP`T2M$3R% zUaLlN|jJC$REUBU+l=YPla|BgBT2ZMYL zEUEt}(w{48LIjdzrPhDX>DF@-n~K&0xm-W>Te}la*JUK;D(e9?vrz2l2DA#*yKb5A zRMAaq!oJl{R1T@*AgtH(Vpw;yUkn zed)fGuB2sJ!ai3>nZj^d2wq&1QPpd2XL(WLQH3m5gk_^ZSG_KE?pZ?>B^l@Q6jyWz zZfpBN9E?x(-{rObq91`==0n28C_Sf6A|WyN^GSf|z6YrG>o1lTS7qHPy2molZf)1+ zH#Uk7pf*B%e8~sw8n>wna_3a%tYl;q+PCVnQQh;?GEg0rmW^Od zS#jQ(4%Ar&*UPSK`XkdFQ?sEP9eVUt!ndl8O`0v^s*ER!mJiUwTDO?HW<)OG1%(=( zbWv6)%W!}ERP`hCZi>=NNg^@vG2JtD3?vSspQ@4#N<_jFUkR6OJv8Cz>L1ighIBrg z>^K@r80zS7Xy^)*`XsTD*nzXWAm%w(+1s5mJmfPz+@Fl{Y0Yo->r6=azBdN#JnAg6 z>#AK@T=5ti?(2~3;9=F)*K)GNR7xmQD5qlkrUA#a5DJmztxVijfHWKNc;=BrM z6DzT?DJMLw+|R-0-{o!pwOhQptU$hTZ!jZ6&er;;I`btTNOQB3wet>?CaN+2)0CWz z^-onA8bn6Mck%HfHrD>hZ!FARv^LH=sx(n&Y@FRSn!Pt_cd;Bab77)4C5xt>?nCng z3o;H$s~Uvz{2U)%uBm6K5RNi~S{A!0cyKFavSyN_SBjGAWIr-+ZubZfyDHXbimLnXmx&so^nD61}w>vCC{Bi2Sc?u2e?O9C|nSZu8(bH2(iE7Bt$PgEO7g)$`KNq?fsu(6BvK~jD#q)eA zbp0~35K-g7i&IKUBF0SDLzg=iy=i7Uc|9_jxz8tK!^Ub>fh|2saWmB`SMWIGwoZ0s z#`?v!Bt#qW=ek&iigT@6vc=oZx-V)vO>~?_6(;XC_T(B*)Q{wvY}iBKu?IKv%^J7! zqrq5w^HqI@adas&R7EEF>(2gfKGU9KnQ+r)=R?|945H+o0%Igo6MI1_=cNL!pt6>4CZi zW%h7!Po@2=H!fr}TeBiU{IEBT$)T+bA|pIB329r(%?-1a^@={O%pQ%Nrr>tv%a#&w zN2goOG7>af=S6yx+ycbxiT!kr9B#Wc%dEJDSV9tD5;SwCncEpq*Uy^@2iF;^J-?k* zQc?bva(HX*!TBz~5m);kzMOPL({ODzui;wM8F)~8dNQg$U1(FU1ac2W|iVH-}`0Z-~f zH|lPpxwgOnx~sG@bke)fiX4oUge%6$Wi3bC^u3#21_ElVc&;p(6M(oKfqD3MIl2FI z`@UQ$UE4EUD~dIY2P8je~5CKCedKg=X6)*qt45Z!`xQv zX5W;}+C3Haz;rW+l(4_qgy)}@S0LU`o0z$KsIwpKgLB%8D9I#RlC>c%Xtc0~A2wPD*Tub?skI+?HAASsdPs4;IZzu$sP`0As?bD6HOLoI66JjGm3XmvQZV+w zblmw4;9iJMyzm)!-nu*K2~E>T=+f6h(cA;7ID&NBE=B>aY3y$Ol}wm60-D9>dC&6eaV0p2T6^h zA42@mw{_GD*fxEt*{+Fu#*Di8%}0+l+GJb!&G)?0EswXTf8cl|Xu?Sdu?!Mhki4j@ z1-BNYE;o9jy#K0j@o-_)s_zFt%2o#+JGJqmu41RQF+~G17B&1vm5;rFa3y`8Y-|`W zmgZX+mtvi&s4O8`PzsW-raGx8-R8~!t{RM{IN9Pm?xL^d(-oWcchJ4GecMcC z1j1ky*s}QJ{g8)nmg$X&0n&aP^$|7cR=jQC1#H1S7t%bNkfFCoY-uk-glWgOUE+b8 z#M@ltQI!MANO6}kEsoKZDk+BT$jC6Y6{IZ%AheUJXD#0}51?*L@1%suLr1x1D1rj0 zlj;-elmzz0sK)>goCM)O{<#2yMA@uPSJj;BJ#VX4LD04bcEu5GRdLz;`6@=0gY5z3 z#_D;N&OA|b zyY>m}CD;^Wpj6fCE7r>{SZ~v(hV*6Efn3v%%-~#;;{C3|07soKho!*iuL{+ZHyEuA zrvcs|?u4y`8$Y__t3_r{2BMW(D$P3A!~bXG_?ssD+oxbU;wYJI{TJDZyg(KFQm8!< z5%RN57lKeIxA5u1Ol zxJQvsZkyOa7N^T6<)Rg<)Uy~3ynF6RfsRLQ{4j96R@RBx5@g8!d@Deot_r7T1K^F8|WK?diYKL->-&Aczjdu}5J z;0eipgwHhH;4%fyHj9R+NzS&}0o}ujSSLE{nS(Cm_ib(eH0rwzoT=G^lif7O{GqD| z^as2~feWby-+8#OPa*Uxhj0EDhjaYkXqR9#Z1972yq)&T;8oKWs;ABB$F%b(aZupS zVxfD7Z=po^rzYh&OhMFg%oD2-LTV|(;Mj~fcZ`2I^*2iMZ~XIL$Mom->NNS~)?V58&bxvf<2ScY$SN2`A} zxBfhDU|1?Qekom>e_)~Sy=jl(-SW%8B{Anuig-bP=f3~HXa6Pl!7GFw7iDA6YD*>g z!p2wmNw9c;DSGFV?<=CubI_yb;`e9d({cMN3;Kq@kk^Hi%C27y;Ln5lZ;mnAgitlY zL=v4Vzx>hOp5O{!d$vYJT7p8)efPb&;F;n@*jUaP<{qfc#T{D)lcimvI-Ys zTo4>!qOtk2pH_e>Y;urqRZ2?vfp)?WjrUvY zQ8IMmdAaMEekx(&RTn{p(E*+a{`x7Cx#uBjH4k&ueWgT?a}5tqKtLv>qw8PpcO2mz z9#eu6Gu=N2*hTf#=&@${zX}_#0-3q^s`aeqhY%$BE@^4@$m2(V62VW%>LjVxo&T)T z`Y$*A2BC!Z%LUN8RMa8kM4qqQ=B}%Jcgn?8nHi0AmUGg1(%NRzLFsie{@EgZli#@6|M$Cg@~_cAW6tpzzcm?8$C?I&r6xj4O7`9- zkoV9mkFW9Q;XcBWdB#iCupfWAtD+lj+NIw|#tjB)kuwIC6et}@vRrc#Cu+D$5A^c% zot#_(|BW+j!PFG?J-yf6pl(}lx{M2ewvxh^tg?GFdJn#jVIPYA2qzhFUEAa=9x0KU z>+U&sb(%@?!4Q7jsZQI9wdM}(p5v}PjlIDKOPQZ_7;_cTncUW}d35{8D0x?JM0k$a zuvHn`AZvCut-oGc%Fl7%a?JBYr@*$B#=F{eT&Rfj%DM>2)C<+V+XZ}vuKGQiB86y6 zxhO~&s|drj*P6gp9<>XRPZ_OmmZ&H}YD~~6EZq#j*qbJvYR`S0T&lB!$^ximi!c-) z8&f@xG*A|hQ}YdME*KZLaHx(n3`grWK4KyJ($>-9pvX~ox=ciqoUZo9?A~f>ln#Ve zBR*q!N#Ol>Ej6w#p;~Fo_iU1R{a~pniSk%xw+xu&ducnndbiYT! zo9!3^VYXU7z^Db$c6g0h5N$OB1a*8!Ob5OZe^fWgtrhijL@NOA?8(nM6;N1f>TIJExYuJ6|*jFAhT7e?VBWhYHZ+l9DXw%Hu;F7LC zzuxe%ZI+{;=o7*SMf48EW$+u}E`x?@4Z`h6jpugJRJJao%0|*!H%GASy{7S9mE*8Z zESMJa=|aiIbi>3-&;G-OO9|fZ*o+ymV0tpNaQMD#QFV_03M{nI^A^^kAe( z*_V(xSUO1~bg0R=WC8l~E4>uK`Yi__%%OVXK7CDT-J8 zfnIv$ky#Ctj2u{&-@n(OvEhLck8BX3#N5S413hP#8e*FqbX(H1_XWN;X!~1^3Qsqq zoxVRDFG`wzKVH=~#40m3o?Pc~iPlGe0>@$h)2-NB7~XIvb5(Qnsj@OhB(OK=Jr4)2 z>-=qYN||;|&Zw|9sT@l%Cq3vvKpN2HvUhFF*Y6!4Jd$VfRKr19WY4@6WPuSAp!xo? ziZ4Vr63dfhRk5j*yk_F$IlGrH?I9;}SNu}Vo!3@QRN8W%kRk^S-0O!Gf)4Bb?oBo4 zR+wLb%;0uaH|?%HaDr14jK_Zs2!iB4lO~3xR>@V}N|ZK86e(N3W`;H2#aRBaK5>ti z)ni~u@K#2gpm?1hYD>CnHe9806Qu%l(-;F&`9SlBLFM?e-~nL?Z?MwCLt*S?Wm~Ht zZgjHo)y`rc(n4%RoMAz*`dDENG%Ac*l99O*xqI!aw{dKVaX}{|1Nh~^SEAxHpOAh1 z$LdMR!$q7ZSy_qxX`Q^?kK8r>qcjX4ztWYp66oA>&sj3!pykEPJ5Kqw)RI==mVS1x z24Oq)Gyc^(^8-m}VD5zzF&kVHi%(8tz6WX*`PB)k^61NeAnf~P)E(4N3^L) z%kcH!@_ku|&pVeFi*JepqCnf;?;dwDO(f8kj-q$l#vd;dJ+l=3!DhGYyP;`Vi@7=O zyM9im#K9;ZX^DEvff2GU+S;FNY}>!({b2Tce;I>EE2dZc+l?x;OUGsJaWjoo?1sg+ zA2LJ|QLoYpg85WL8aA7Apb6XQI`%Uce42}oIF#UJJU_i`YWPC}s@EF^JG>x+5jw6| zP15Hb%@8N?I)3hyxvzFI!i6=vdTw$z(pL&#&95AaY4vtVdv70>Xw|^n+}0#ZWY4qs zVJq-s>k~-#_qE02l}=%o7N%MZ(YUf&yY1{mpYP{lJ8sh| z8t4o5V^t^6^tX*^_&{%VlSq5S^r&su;_pMOLH z%6E9;O_s6ZRlt8txnO-LAhrlb>ea4IZoD_i=U)2ZU-9gyvH2MYYepnF}pZ zhYX+s7iLRLudRJbOd5_&n&o&}nkt(!8>6wQI?{6`Zw|N9yN7xzSFw63$JVHiw_|*1 z4l9WQAxY+=CQs+_n9}j>`N?Sp|nAd(f%&{rdzxE_Lhr{8$o*C(%* zn(_WiSf5boOac4DX*5s4=Utr)4}`qNI94cN+JXzyYn9fYUER`v0G~Q+WZL$T>VRrm z80XiWDNGozABO778_689?bN+OKfp{ZK1^HGJ#wy|`=~IHWbbgQX>u_>(Jt-6ifMu~ z&T~$U`qB<&gb|Y{Lo#hWq;N?EC&}@3WTf6*>3O5iq3H}k%+r(l!03J&e zqxZ5=dP(ZoJSU)R7gwA>l&qnYDs~_ny_UQJmi14WpES zlMU#f3f!5OOG>vocs(Xiq|Y-c3|Hv{`+ig*?+ln(_C(``YjO3?w+fy2sd2$&SZbR_ zFgFTizkK8Q$CB(XE&P)y$#&d+|PGN+`up8Nn!a(B@Kgg+2&*rh_yg+pauTIl|P4^}+JE}6<093kJr>p~NT)BaIjhi_P$=&l= zH9}h@ulFY&)o6n!sm_1~kGkl$arwF{rlR#n=g)5!QmLI{it^67|LN52aXZ|g_eZq$ z=ff`g%^^>$>_YSThb&=0+;H}!typoMNte)wTYq8)`hnX(SxMD&ODl6R|G+Fa*f2_)f#HVh?@*lI{+C*f}im-Wla=J>>Y+hb{X zwAXVKnuk-=J?yL)VX-0tx9oL9YSkeW!64tKL~n?L`u(g>%+_`tw@({UkEm{cqlW6Y7yXa;JtOp5y%N!{aUuW(hPv=e@Js zx5yDtbN3iCPt_j&pIh>iOl(3AbOLaQ$FDYaK4HX2yO`(qZ@DGag?A3f)K;JSXseMa z>Q8@u+j=Ike3Jt@Cx>f42wazNu)YJx-8>4)-lkQ%9 z0216$7#LaOc>)Qg01bMPH-a_d^NW!hw+c(47uP+L1%*MnA>@NsiR4G23x_H7t`Mej z#iu3J^`hU8kC$59B&V9+pTs&?_}lKm8@0^E<)+0C?A<}W;sdzHnC-doxVil%bcX)~ z3CW5BS=XfL84(`^-CJ1L7K(jh1D3-klQHIuP$xrFht&k?Oa$g*{n(JkVCJ7be>jhw zfx=|iLGumd2HN`^Fe@%^!xmrIa<%?8a8nB<<1v%>e)W%Z@8fLOad-Z>CRN2bB2vS% z)@!iC+vqz~8yW9ytd?n_TB1OkRJRixL~3q97=RIKp%zILBJ~+MM~q#S;BObeoq1Bw z!8a=BrAf>__GtBk%p9!S_w^^E!G7l#y8~xK%1(jucS|H`4_lPbEb}XxyIoD1(54yM zDWHE{=i(iwBCC2g+uThiGvNzknbXz0-`x?VxpN9))gC`7XX6boh?I!DkL|^#--gik zckAj;JftFO>Lq_Nk$@+*>dGq%pSWCf(Ij0uRpL?MV+4%K5V7es3sefK;)btTL)~_=j-T4`!;7lUsrvrUL#xmI{nsg~Oul!`)d+(Q`_m(qUN^{0c#T}q~oudUmx0kuD(J?q*Zg82bDKN+& z4!RZPb=@KNYpLh{L*Q0%B5at{3a-mM2<-{aIoRC!LC%9m>d2k@xU+&`0Tm@Ud={26 zn>5cqjQrG29W;@C=Y1La=#5!MT`(C58MCRj4$9(&239!4ak(X=CHorc-6IlJz??#b zjZi3^9`b5ZkVEPnRbmBERKcwbYjxkG`glFO_0q$GQ`*6IpQX4+jX^b~tUyH#Vxw5!RCdG6i>K`_u^gr@ z>2c|_ngu9H88AEJ?;)Lxd6x46q^3xMXac%%Nf-Hl(DOb#{a{b}ob>!*`uBz*u~=K2 zBj*aptPc$@muZ1IP4&1EW`rD*ojtlv6=`mK1wUjErM4l41j*VMKcn?krSina|m!;MLqZIe~Dv_ve;*C6^LhVEH z^jqS|g>q(|Y_Z;NaAhJBhBPH%i3vjmj$WaKfG=iqIVEq~=nu7ZQ=w_;9T_fr703$S zyp@dGiY96b1^*P;@9EB=-#<;42hL2EJ#2F^tUWsC-~w>sTc5Nfn8tVfcRyjGP0Qkv z4S17jiwdly?OaYZr4M_`U}KKyxE_(@NJ4M=c4@q?G_IuH)@8^0(PIRK4sOfZ%xGDA=@ z`l2dNKGss7?g~Bq%BQ;~hp6fQvG?8qHMVaY_;ZevN@yxHt%`<7O7oam4MJ%Xg|@V` z&Jj`?yhhp?mDQlV41DI7e4;H+QoG79z98%~|3R_RFW2un7;~EOY!&qg51Ne8b1Qnf z{Jc2DHz`MQB7{21FR|1ux0gA%Q`sjjTWxoi+tEc-N2<*8>f3^3?iBs=d+?oNo?=Mb zl1feA{N6)F()5$B44U~cy_sdE?r>2Gel;x`3qE*u7NB(((rMC9)@1C47cdJDKdVFs zD(l=UNJxwsZ#nv&d9l}$4Mls8jFd+XNq?O5yHDA3a+_7t1lRK~ukX{c z1AMwtk&Zi#NJ}r;<}n`rdSca1TY+~I$_tDu>o;YL2fmSBevGWhrY>m9f8kESN*w+A z)c4qk&cqf!qv3Y@?D3~gVMEVqPHIJr@BI9k{?5~$R*8wl+{hr~`_U_Fvd9*+w>|o*j3^>s=L( zea~d~RJ_woXqc=Z2MIaZCCfGS>&S&{Q}_DL+bxppdwMvIEgoMJuijfk$-ec4Y|$mB zZx=2ev`|A**MC*aNM`oKtEsgy)Xby{>Zulb093OUrv1C*9Z7(1J7L-zAcNPH!TcEBcFEge#n3}8d+$~l${rF*U>^^ z0quf}pdqXhs6`CtyLw|`c3?oIvz6(B z>vw`Y3m-`O%f<_eJ;^#-QObKe?%hP`fmBaLw-36>8OK{3Y9vp`oTF?=e45l@yl@Lf zq1C`V&ZrARddi)+ebpC|k*OteBT!8%B>8i2~3#4^(XyXd~+ zV%$O)pLI_%Z5$Znlf{le?6Qo_D{;N>d&?sI{NhWU+H2I5J3bHaoqHD%YZiKl z`;qngpwiQ$b*-7U+fps;h9zu2i0-m0ygZ)ON3qa5s~o@b?1_v7eg7f1dQIvVf?3(= z$Mwe_vgovH*S20vAG1BwE^lwrm7aRU%(eVx?Zghn@)Ly}1z5A%&SPOpqqhTiGdmPd z)!cc?xTb9KlZQdd(@%-rgoRr)ZJrr3EiPr2T5^^$aXUR_Bx8uW!1Hj$*It*W7n7yR z57*Xxa*nO+_CDoVci>i?iJoR}FQeFGC|k$HtwBb*ji&Ak|KWU~xH;2}fv6ON+)stR z`Bj}3$)F=wrDO&lF{u0RYiIm=NDQxcn3;$@zGOVDJZZ%BfpzhX*E1LV$kDX8dvKi- zyx-_H4cHx-aZN^k_TZrIOBcaKK>~{DOW6d9cbKm|L`XY$GPYQ-(Pe0j~Kud^JM>bD@NgSaelp`M;kzx34>|HSS!BfenG7sUjU+QB%$h!i9 znEc-m++}k-mas54jDmza^A&7xVhgSEOYJN?e{>9&# zn6$Iknx_7UBJqPxe z;qd-6I6~LU2jQtun?6YEm7f|`_z9MtCEuTt{_zR!Q$vdahn>)L6W-U3UanSEe8}v_ zrOrQo`NwhMXf530!zM4eL4T5IaV^dC+QTVO*xV0;MqHSNQ)m`gZGrvEs%bcdt$6y+ z_jS%W$ZA^U*}HhtbMzKg2U?r^iQKQ+@r#zv8ZX?5wtPD^h2q0NKh*uva}k%A2$Az_ zHSy8#Yq!z2c>apX{M($rbRtD*>reO+&;|G9e-@kS2j1&h+1U0+cF!VxY8U}Mn(37s zmE>6hdBxr-{fXNdl4mi`W*h@;fHfBuKSNa>dX-Zb%%Nu^exK|Y!fD^Oq09eI9YZQN zaMRnE(RT^n?zgOYV62Dr(bVPXyJlZsnAjPUzO}|{io+!qhVz>NFR%%vP-Ou@aB}3^ zEwyuXE+eHmp6sPnRYA)IR-!V`StlVY`?R|%xihsjwB!{%fAtpJ859GzT+=dMZ(i$| zs*_I5qNYSn7_PQaQwiMigsl9Y6OGcBnarw^R5oT$zFaC(Pq-?Uh~5!((BVb#Z~pCX zD+=F_w(HL#eGMM`W$<5NzT0iMaO~7!Qa*0<{TjcvYioSZI807$-2+6oFJ1Y!fqs!d z+L7r|?|bllzNySt8F6_F`pU88>G*rr#ppeSR$VS1IOiiqkhBg>56p)IC%}itEQ713 z67`hd%u|be8NsnHSCB(dM;owkJ>$apu^k~Eql0^`HoW_ex}U}Hq!Iq~t-m8AT8?trU$?N^@bi$S=sI6 z3r8l|)M_)WEDIHQ79SPT{&vUDXJ*}T?6Tz?W%qA+G`)iglu_xGBV{BrRdFUPBfh5o)SyIaL+q+4BW%y6# z6mJE3j2;Z)kmTsHOMc>I+X2J#e`~0W1(u{{X!C4#|w&66$o@S{X=X2hbrBi4D> zMjrc+Q0)Y*k^ST69ETO8gYC`tePb_g_m|H9l(XT!(A{hU8|stqudVZ3p9C5E-+kCHse zy(uJ@ks6@^>K>{upzs2y$>V;A;j@#XzZ`v^LqBi$B8HB86q9!*#~u&&6P10;Gi+mE zyTCjqgt*mO-4|L5S2|T`2ADBI zv_9g2R|{XpThOqDRtjPo$`o#>C;}aXc)i7Lp>3MG5$KbO#xV zCndp!Qq~|c$F`UKyx)3Gr~gvu#%m<}n|#MeKu6`|!Q8a?ky1tRl%zKyx!Af1Yx4K2 z+XA(MF?~-?m-=zp?Ie~%WitRH;bg+1?2Qishnd9qU%`BQ4{&P*+uqZN`rAJJb&rL% zHDv9kKU-gEW4pKJp|gr#dPCO!`qIR~sr5kZ@j;;osg~ zFvI-c<@0EtKm-O~vioi2_{YqlQqbYS9h>S$ZawW0rJvXzTWZmiayt79m)ituK`9|r z&tb14|M;hPS^Knf)op7>8G9FBKD&Y2mkv%Sh~bgAng?b-Pl5bVATv&IZ#Lea9x18Y z&a(<~rkxN}wvGop{Pp(hzl!)P_{F%8Kr<#7?V2$7$kja0=IJ6=a6-gwC~QSvGPey( z{#FTB!qgeqIjNmei8fB~gf*Wv65z!}Ed-o(S2~C=B0Lymncri}^9keUQHh&WHy zyJa;2FN>yx@dn#|oQIuTRsyOZC+m)v=4kYN1na(`KI@j0xf!Mo>alkdN)#gtaEYKA z&iK2<{F!3dvqQ4{a_K{K!;TWr-lggL<93WWT?)%J4d6|1ni-nsGoT@VEywkF>J`In z;LYL1KqDRt&P6bs{mgG~2L78%EBH+F~>6>xA!e*xgah-->oXyvK)Ppy*sm_R(emG>6@k`bX%IgtKIs> zyf_RveAv53V(><1cMxXn-Ea4uN8dz?kHf2}`3+YXF?VN<)T+F3GZvXz#u2!5`p=5} ztNs)#pgYR@;0Jf^Ie9FQ{VLE2wm`#-!4a(4+jkQO_3iSQFuxkl-Pn+ z+qzKBq704udGN5Ejn>(OFd_!1iV=54}TXT3&k zE=Fk4)_>-SWF^$L9_dWgwj^q#az4#V0q|+G+Fs-{W0{puBDg`T?Rm)h*UU5h{5yo^ zH%9;QuQUSy&l$ycW<7+p=~Gs(=E~MQAzPDyx+K*uW zfA{zQfcx`?OZz6(Hh%Gu)zow`3-juF#yv>R>%Boc0^`&?R96$rWO<`7vt#*DG2R&E zU~|Rk^GeLgiw>ZYN}}~tXob~z#hrKJptqY_knn?#drf+aBcfC_w{^5;3^!7ZdMWM- zBAs%t+%6aJYg}I;!6Lii#7#b4VOhGy2c)$!fp_nTCQGpCdCLSGx>Xu^cJork+p5psu40^$HsX{$ z;^gMsI}+SFA>Zm~d*^6-RjYjKfTK&OVn{%Si)t`edZUVw5|!c}=EZ-4zjG+*@S{jGd{B$zF~Gh~rNS2Mpi7kR8# zS$C7{i{6QTQV747MO9Eo!Fl1EcUv&KT9vd)@#9>w67wN)Fh^$w|SoP7A{f_Po+hs#wwF;xsKZFi^W zG)x}$w)i)h7EClUy|?%EI4N11W>A!!7Oc43ZC5)!eocvTXY~7%ha`jERs&zIA*BYn z#hu;DuTTWt>e7b|dH9&=eCd}^UvN=?-iu>4q+1QIua$-g>ZjL!JA2tCM75&Z<*^(V zmn3kb@roFda%FqU<+*r9W9JP|yqEmG1sKvbSBIVtd5vXZ@{$j@zN8B+45RQ;<*k%R zBbg~(W#zd6xvy9;OXfO zXT;GVaw*mcyXMqkl)?+g1a^l$x{uULwApu*_K$?|y0+NM+|{q>g%@&Ve2Zs(d%ad^ z@xsCZ#`@Zon`D+)jY*P9%5)ONwk|?^#j8I`h18g?T!S}MC30*u!;rRle6#zz(af*} zzXM(PJ1l`5UiXHTxi>}Bg8V^J^WL7Y9`cK4xvFj4p=rrR7rg4e^+sLpAMcdsNWZ%3 z^(zxM12!iM7nk4_w?>o)l@3gLE~kur9V!dnk#hE$M8h`)FND~Ln_B$!D$YfMb6YSF zvw?H(O`)@*$nrPgb>E*j=eY;56?Rf9N2Bc5G%gVd8}y+FU18p%>x5d32a^g8@$WTQ z%DS{?TgGH{f6rb^XQUFxfbyXQOSsPR z{UEUI)at3H?|q27vwh|_e_uoeD4V<|QqQ@tfejuu6QoUD$uYTYhS7f~L4Y5oX2X!2 zTcY9NMVx#4Uvna&4(`Gse}870q{phDGH?PR%{<}+7Wb89>QOVJ^JQrL;~z68thupT zZ8cI+*!6rr0e>31UcUlMxMRMdo9!R|7K7c5dNKxtGjlhG8UZg(do=Z+4(~s$93d9a z9safthe(IHi-60FdqVT{kw#XSf6xnIK!BUZDYJLcELhzBX-+wMJe~)qRA`(^(u**_ z0;rkBsnZuY*Uf`dt7)A2*d_{`GN&y!v3mpg-<(=U$p4diU-5J_6pBw%k&4ktOWQ*;6(ue!p~}aY4{M=h3XzCUWfh1wEsBmcIPk zhWQ)EnDym_M|%~yEft$un?@Q3}E>a9`9M3KSqb~(ME&^cW!{v?efIXj1Bu(-7KQx;lLkoiyqtHa)Q4C%{a z&J++I^JCq_cR5c6sR;Y@w77N$)WTD+|cIPQZ#j*Y+a{@4!FH)_Ru zaz2_^2r0fue_K=%Lq6E^@vB1K&<>YpZm!3ihsBiGl&IOUWxe#B`e_s)g0z!e&bhF1 z-W8z|6>j-0VH~x>$Z~7vh?ZszM5uKW`SLqvx9@NIiS}x)DUOLc$0yVi=hWqVn(@fw z?b1C!K!~=qpkS_qm14?=n8}_QFEV+q~g zg^#^9>xO>wYkSjoN_<^aUF;vYJ$kk-MSMMklsn<5#bolK-nmFf+AyV*DWzvqd_nuU z86G*DU_!oL!ky!+5UuBAy%iQ4B3~JsEA3iPry^%esfO5yVcRE-9(d5FKGS@P@kh>-d7^~8# zYq^T$a-8_;4k1Fa4YvL%AGa*V%AB{PwSk=!Ku$aaz>L2Nm`F!+vEmNz`5+ic)ATF- zh319rhECXcdc^SL-ELEnR)lt(Wa_EG8;Dauw7 z?@_P_VP{BQiSQ^zSgos&9`0(6JK-dRsK#zbtehq)^!`c&(AU8R2@EUp+*GaoE<(IZ zBSU9p=xIidRdK^xvd^C0NI&-OlD&I&E>-*ANYm_Ka{VkHNimsN<1&dw;d?GkAiXUvEwa`HRATGvmkb$_?h)aX9x z4m)*q+2uOH*KT=3zF{rp)P{<)m)=|#EbmZgRpObrnexbfyj~WQRvX<(kI*w&>Kp9@ z9`P1B6kA;&BK5<=FVg4Q41HNQWsX|qp`XPt@b0`RYfRcApZD)*iqmF+rBoh9`T|Ts zJ#+cp4JsxwI0R;V(&0)CGq{&k#^ja>Tlc-(y z*Tl075>p$U=(9ThbJ*kbSH&6Ab3hMat?IVs+vvN!26(k^!uJ78JW8(ep%*)z$g~FA zY<<239amzY;DP*H5~k@T>E86X6Fuo0#nakW3sEZraSTV+7t1mBQ6?Z_Zp?CcL(xEY zB+sfZ1^zoJU$hKm_ZGyVH*@4&gulMF@coxy{Sw)vW3}#6`Qkv)=S6c_1x*o%M`(`d zNo4yr90P}l0HW2^#-aLed1&1uAoZ`?6z&0QM6g(b-GVth!OT~r=kUvTuT(VE5o3TB zIN{vBe{&C?ONT?`6b-wt;rr7Th1ao`4VrLkf<(&xyr7UB-euw;Nn3 z^oO7Qwo)@4JYrRK!2+^$ybTara9rE-?{myNtbQHH61!ChGnO@g>V1`8R*kfOWxF3f@L{=nhoh!mj2cIR27?7>k&I0C%!AujlnSQ=! z%|Cg;@C8-9-Y#3n#rFWME0*kB|4&{Z#2r8u(~8@h8ClwrKm^IH&!E!sFBamtVFL5V z+P8p(l0a-BL(BPpKLw{bkOW2;`vwtaTDVgk`)|I7OMqNoFO1Yh(^>$Z6{L1}{ac%g zBk@8D2z4@jFH<^V75NM%at;S4ozAq(d1Xo!uiadtpmThZ+q&#la7k#-ghWk zXW%XEOU>sK!iO^er&hmvv=>{&>CiV^W4*JQQ#nS?t$U?1b50L8vlw-g%)^2kZ$zC|(Vm9dRSv28WcAOm0v^~=xW74!gNYjkml*?O@ zt1Y0O0h-gl<_qN7biQsU40LV!f><>VLgng>Tkf&f&UM>3CTd*;POipPFjIM}xb3U6 zI5N-F8y2u&6ME;?YV}lnKZdkDuGb|GljbTB!}Xk-DL-1F#&UTKH`73LrtwE1+X-TJ z=bB;W1$J-h`1zQZ46C(FXg$A-MB$LAQt7gtIkH1OdL^;a?_OUc`*cZ??oeIIT=THX zzC50_Zd=C_okn`v$R=e5G!Kqv#O5{?+m`1Lm9HkoK&rBw-mZ?tL(o-`VWiu;%D2mh z;z)eMj`g?L#^$tSvdu4i}CVT9gXd{aWPi&-?hJ13{89EbJL ziyay8K6HD}jCcQw^IF1mn-}Q{OX4zB3`>VJ^5)m^uu?rJb4IypD)y&OuBd;q>eV%^ zqveC3W|Rw-Is3zVy(uQnv2Jg_wYxTzXPoaE?`Ni$!^AaY^WR+~anw+o!78-Rpm!CP zu$7xA&h*?yyY5@7n=+0)3==vz+1s``4Ab*H^b_~ojweY_{Q_b&zR!`B9LDe3)Q2OT zuE9SqIOC{UlyhHlF(~M^M#0VQAlu8U;JBDr26YZ z`(?x=+`FBGfvePuQj3rzsr8*~@3$MiLkt%yE^#(1;(bB=$ooPcvFbZpZ}eiMko@Ap z#icG!(Xcd^OE}EL!Hej+e6C$o^R#3#(pAvGyxy{akwDg%}=(wm@$xNWK(y;J8M z!7EZ^zDKiYC{@wTNDv$#U8ja2Q)mSs^9wFg3#!)1y33~?jRxRBrx zf22O+QjgG$c>D?s>7lY;i}&NLekaL$8cOW)lN^10TS)pxEH)!DQijsbwG9pHoi|OobQ$ob=(xz%-A3xO z?lEx2WEqdQUK4ulQiApB4B4yo-uQ?hF%a?>)6lHi^okcY3aQn9;GFiA92OxSrz>Mbmb$yjS%T^2Qkf32D=AT~xL6mw3IVIhw5C)% zET0&AUOJV={Y}`Uz2WfA-37_PyqYfOLjp1A%~+ zr5Khn5z-%MK>l1GNKx46Q$$WL&rvbpn{3m|o;XuIxMAWFegzi2{4O6>vt6dQDKlebLFr$?I$Zg!wm%D2~Bz!;tLX*7((OJbUQMoKs+8uq-UCt7WvqGJD-c16&Bj z)T-u%pR1IjOKIQJ^{ZalMMSHHZzHP0D=M4rSw}nx6R36Y+5XsSa&7jCuy#mdWYH|+ z!;5oXBy~e_T&+OYNP*VM5Ufo9Sluh^n+g8o+KO~mRt5(zV~Ol@B&he)zy%xjQWJ(# z9anQOD;EK*Dc(`~L$(=tvE{@u4~-9Q6bt;dduFa|!kRaRXW zvfapTzAod*md^VoyG*m#vPVic^{J?_=oVYa-zCYS6qqf+kQ@iA4-iPOD+%xZi^Kk* zs@FN&;JQ<*K!cou5m(vTmuprFD)n~aNV_d1i!8MOZg@gH5At_IIT*X@kPY+S1UNm< zmi5I~ka~f>ktYu2ZS2`_pM?dh>1j00Zcx9yi-325VTKBHbAQbL#}DsS@JL@|ia_%H z#~vg>cq9qk(r#vfJ%S;_{yBsNU*VgvdJM=o&Z;>m<@GQfBEpk)7wVd7;7GCq9$)*b z22L#?k@3PRWRxUkl6y4FS+knl*0)6|42d!4)+~=j3~kqxOJK0uAR}y%Std32t|nZZ z*tmd@tHjltF&KG`rnhI^8yJzYL{E{fV@TZV`=$X1aT3&IAjs3}9~n&$z7j*CZ)}N= z^-|NbvvD3uu48*M8U~k^z#?7$PcQN|I})rIQ0^NxDV>(XNJunq>KI>!Jh%9^QV=ho z4B017)C0l42MbgOzN}NqO`63uMa7~p~$)Saw=t+TNNZrvQ!zL~EUv9lne zW`-Zv;lqP~|whN+rxe+(}PN&0)=EhaiK$t}Z^0EgXv$Cv1^c{C~7X;>3`q zt0@oA5>QoFDY~_Jf4IKVB+O@`7m9{iqm4^C5A?sL3@2yvnuIzU-&KXZ_TT;S%vV}s zPzL*csWr@vGUijy@A&PeO!IAxlkk?a7J#P{J8-`IN_=VxLq~#b3b%0&hoWtLt(xHy zu#I z+aE2lR7E?F)#H=8`{qG^!t^YBJ2v7*N1Oa|0mRB%1Sr-Adh&3j4Ky~`>NTS^AWw5w zm;}E#3(4_86xCtaX*7cTYYpISjL@2q&LU)M+082;?czhUwzjVYpE+yAB@)#5Qm;JI z-SKxyZ#*MCi3$xgJ=+1bkKMFJ2`BxTvrwz0HMO3bD7D|7+id}>g$-yW?Mash3s6@% zRnpEkEdGbLBwfPsWq$ME9Q;=5r!XN>SMXrzZWiI~w6QLSgu@2qB^PZhS8ibxojiwF zqrp0p_L!akF#lBNHEF9N6T5)=Em}oiMQ(rW91p`+T4L(jNFPzB(D#|*3ReFb9s_cB za?V*^zBYFIY)SorXL3zF%9dVC#_a>0(z%sxlN$YPK9V(wh3tISf{L;>C=T_cOKbId zj#g(osZ0OEa5gXms*AAuoy;F((xH#d}`6Moc z=W4%x)*(a7yQ@v$sK9OYmj0r*avL+(XC<5<#Ay~;B^jM~)jgziz_M9C2yJm|^-$EI z55Iy|YBt_t6*1$u{_Ld$Uf>TwcJRT@Qh`&t&dFTS5>h6zCcl+FAm!vv{1h5lq;sc2uoNhMR%`KnDn&H8)8?D$X ze_DL&q^h_- z=-l4-Exn=Wm`9zluG=I#=^^{jyQHp<8`0#tMmV2BJ)pArRu9X*;UrIM?{+9BgUjN{js5vzuRn2z?As#oF#j$Dl z`77DXlkc!s-mv!_Q#>x8H!i1n$x}o119wPF^2^~tc7)#6haMZ{{MEi-pNrA^+kw>6WnK?KWJ;=5qaX;Y$eSr>ow$UCx#?D%%b+04im2~Lth!sRqC5&C z-L~3WbvJ}j&N+`iEjc+Jb()@-(&l!_?k=JBqbGr^mmt_xKGE>)V*K-LHY8Yo4O$obl6!#*eaFA_3+H(m$yH;Ef%BdHod>1|)?fK2- zUfTl_b=YlkCs);HjRtJ?s1kW^1FtaZJ*R#ZejR%nN)aB#d%_a*wvoll7)N~6EXElf zpP8g*67=1CI2RC}o#Rn788gX)j#4-5aj)e``;3g)laX&PJ~3X;D>r+40Ff9sgGjeSoIWVJIfgh`xhzZaJWi=nH{A=Vjc?1-zegj*+) zKipTjT6eN}AltMx{SJHg<|T-)qMp22Myt@vr5`3)A9DPjmZj=v@|;l;48EhancjDz~P9C|&u__Ym+AG(EdxA#VQb=7H$T zJn(6w)fEvwoVEyrsA6&h|@Qoj-meEa;)k<eFn)11Os-yXI*w9DyYQ3>{n>D#R~1{1BO=e{%62h_?;c`HeuoVPIAJ@F}o zz?qTXvr#>c<#dRfOHrYCQdAU{jzWHAM1D$#F!6-`H&Y+v<$AA($Sl69A{SuMmwl>^ z$0U150}xxi30fPw%9s~XhU!m|vbiV1)O1379d3w6x-PY6+3GywIZuwqw~XTY-gU;> zoNs75LRzAT;n8?)OHR@xJ7==g9aBA64ToCag34!c9hmYq&XVV)K}zE^-fjK8c2m3? z(m`?6#I8^rk{24*F+-;w=X(Wkv|ew|w^?+HUd1Hl?Fl1?Op^ES?W1jrpWdmj@_(m_ z33e|0CSNrf?3WN>kG)lW~W>w zfBAK0+fpot#!XG)pna->x3)?~>YXj`OkthIjK5icjt|mo)%^GR2`E0CN)A3(slbTb zRZuFO(S$V78}^8mNK}!}>r)0^%>)DQ*XJL8fL7`;<88W5WAsh0(rTH7%G&4_*G-(s zXo=2UDTA>}?^GLJMhG9VdR60qp~RH3)uJ zF<<;?_`jG&>W?%-;026y(#mg5F+gnrxn{V){Ur`GXbQ2{4QIB0q3CboKz}44L|am@ zU<@?Dyl5SNQ9CE$3@nk}vW{1}TJqdrb%j$aMN2mG(!0djB`M-U*raje6LviWar>Ot z_MUNk=z5RR`ufK*-)@1vz6NcLDk^!hxWHv?9#r3MA2|yHFJ4ybNv4U$K=)O+3eot| zS!B1x!4f?@bk4x5gRU%9t(?82_v%x5n8ce`BAy8GJ8ju{Ra_fl@bzuP3YVt)T4~LD zyE3i5^!CeKsFAWZY&Ez3mNp=QFJ3+E4W~rX0vn&;qN17t*qv>)Jhyu`-@edbCt967 zpTRbLXij&l?B?sNXLcD{eXT1@>3(0k+;2R3WMB{9Iab(jO@7X$$hvR1jW>)YS!VV& zYtdFi#6o3{iAOv40AwDcgv8m!9^z$MXO)q@;}9T7HE95#^a*8?=0yixDfVAHix7V~ zjXRbq^-6OS6@chA3E_&t>z3K$`P1d2exd0y`pf`+gV%`P*0>O zNU6@j0UX1Ybu`@6mxSj#p++=>C48c8>3cyFuM4F$v*T!Npc|{m90``_!TL@E>iFx> z?pHVlQ5t~J%6eev6zo;k(ni0(=i(4a%{cW6+Od%+alc7F{vjN)!NA~!aq zASd_v=GOgO%R`XS!Z1hc_UOm7ID14$V!Tf5nW^rZN~Xy&y>erNhj1G6j527*qZdeh z-DH1k&MT#r?xQcBHMv`63+Pw)`WyuhdgBf)pFOrlb1^Xs|5Oc8y7+;aRp>?tDY@?A zfyvP=nA-jS-ner_l&u1}~=${*^W56$>Bn4X?h6@@rod|ML);!I`Lw%i<(o#>;J*6s> z%rML3DkZLSmEs{2+8m>d4cVNwVk+>iB3u@g-rhE1nahQ}_1P|`tQtx7bs{n~_g`?< z!R_?@Yr1q7UmDxw%xS;pO;d|VlLFFe$s2x!&^@>j7 z7;d9z)6Y4GsED$1tTn>6lJ)8}+*tpC1`=&x2C-@@@q`uXKW zPHw}^8nM3q&ciV>^1%&HC#e0z(R}3|FAonlyV@$n<-Ia>WA4pn_IKEXb!e;XQg|@# zyNE}DCEZe<)2X+b|3Dt~^y?<;wlvd5Qejo&dt1e^@=qU3XAr^a&q-|;pmKdux#GiX z0BUb5dl1Bih+cH|5`qm2sjaS0zvAOY>^4RQYV(#QiQeF^y6Lb(sQcZ*f=s%~TeW&L z##D^n$|fWACj7q5hP4uLBmbRXh~uZoDJH$EqwJgfD6}+J_?t8XeJGb^HkH@0wtQQ_s{qUf`bV9k^l$B^*H-19UwwTV{tP-!hK`~6vtVx>;=5Io;^YX zL#~@0(+mJ?2;lX4IN0zh)wx8GH)C^X>8dQL^5F%Sncv2^3aHn|2AfA8#@295ZBn~h zYCj#7#_?si*jHN>J@!XStecMM&Di;LNtm^ zYV?--zU~H3evhWrR)wole+Ap^K#B$5AHq8bz-27XmMz{3sY;}>SwWcu9hjrD!QR6piIA!l0?(xBhOwzuqVM}|+r^amF z4E3x6G2PfZrDzhLBowaBj#j)NlCC~0CJPmZ@cIVZgbNsw@6uyB;4#JP>-yg96WdF` zdw`ha&;Ma(1-nGBb&16_bu zWk{HIyk&#W0Y>>F1`#iUMs^0+?h9GOe0xi@>^Uel8WpCk#T&X}s8@?D&(NXO4gjW@ zl*z7>SPwS1*$t6cC`7JVrkP7$=ej+^>dRZ%ur^+o&-Oz?&YEGI%O6TnH=i^OL-ct$ zd0^x2Qd|b*EPNF${+g?O8VB~9os#XF!2Pv(jg~&u5)+U&RQ@*Y&vz2=F)(hC z?PjPy{jB|-#ZBSgOY%Q!zyYEZn$hP<-yEJDbZ>W=Aad&qJqByeG%B58exeh+8tpe48*#?$Qz*n>bHNyNM|^~~BM$m?N8AD! zs$k7xvN;0-bHe>|1&*NIt)FZ*Vz5$-Fl6M}&N~Rv5zMG{*##xyc^W`IuZDjEcSgtl`tZ5*x_}8K&@EN@0IA0=;3U(D5p_6o0n>_Ju=*%C zk8JoJiZ;^UK9amjK)(3uQ>j+4u3+V&OsFWV1xpW%jMl{BX4ZlGh$84%5m9@$$H55E zo0e!wu1q=nlRy>$^~Yd1_uV@kDqk<)<+lRwk8sLtS;sCA#{!14@p8>hMi${CkfIg8 zS|5a8K+E`+EdIy_%baq~2}QGq!yD>@U>xac>aLQKshzJVFS*Eh{ANaurfm0)*Hfqy zRG*EZIYE4ifJFJO;`Sl- zima)$vVOR=NfpE;%0vTYTg{Rj01aaS{s+Bw%bkWC$YjHa+XJ8G=zWI9MC$NO(}@x1 zk<75*F@0%}b+{lb`Trj8A0qXY24fbva;5s~v9)(14(^97ny5BN`xASgGaNTz10^}D zeQsUxb6Q3mFB!iQgS|sb?2o0TCaBDI%nRu*Yp={nIAVluZ4PNaUWMuXcvV0Qyh1A) z29!=Nfir61Ab9tCcUP;U&`sEdoe0{I3g9`~;e>mwT}OA%9`ZljW$pce#}pD5A!U-C zhh>LiL{A^?0AK$J=C~e8XoZsSe=y=yf8Cg796=!guqk?cgq9wCE{HJ4vi8Fji06N} zZGqxzBqj082ib*9*>Tx7heUUTGZG~!d0CXr)X8E4sTaXg4EbX9dfTM$)4@o)>SL6g z((2mX(po%3+O^=0A(o0P60`1Ln#|A_jpS^C94IS#k`25;MfZIxa52-mbGGr^FS-+e zRZZMrW1$!Om64zqX5~nB)VwP#xeU~PIYtN(e^&$EMp%QyNH|@nQi=;ip}(w%*~!CY zAnuDaWA8{_@k=WqUrgZ3W}s6WJhB<4^gkO$2hGB_t;TS^45at`OojG3yrBiCN-Com z{_p2Ae#Tb}_Vgl##M|4FV>HE(iXle-@-X|;j*=RiFxcy0_`2TdQEUS0k-+o-8x1L< zAkDc4-McLXtkS13ecXKEEGbR_DiiZn&Ja1l=ydRj-*iFSZ2D87XXyq1BC;~+7?>f7 z5VhFBG;db!5IbEM-FXy3uFLD@p;o=)wfRR55=bXuM6RZuClI`#X|L5%eMAMMJ5_=a z6g<5IP^+{a#B}Yc{!WF6z$n=Nj;Jz{N;N9lg0XY%u)Np^}Nj06}_EZvChf2 z$Y-DaY*=!x5kOLw+p9q{@gPjKxi#_}R)t}z-m!JldQ)(hnY5reRJC}vpn#m^SG0R% z$y_b^d653HrHso_BeHKJh8hpb0;at-SjbNl?R5IQ1Y|rl{M}UPBe3@|rUlS|C3*l> z{t-u5gw+i>&aIETu+Cn$J}mS5XNP{FrM4Ldy6>z8#(sSCWgmg`04$qJ0aFA>!?i&A ze~}T#um!ZSx5`OA1NaN=xSL@_(1EJiW1ms4uPPUpN(qc$*din_Ssom5r-cicTUQL| z>MyHcg4|$RI8h}DhF&0}eFg7H3j*tU7E3`+8bV`;oslNX?7U=(csPR!?ndT_7^^_B z7wDad7mdSn9({?R!r_zjPUGM_9;JKZMKN=hm$2gVZlUMP^`{-@#3&k z-UE?oh4M2N;X`OvdjLb4>N~nk_B)KXM>WB9sv@5b7r<)o?fYMFex~#s;CxU*ECG}d z0@P1r@R?mO9qXR!EW(G;=}Hj8jguqAJDJ}eHwjr56=J6y?9k}B`{af|(2aNo`H8N0 z=-aC082Kr1rUO=g3+^7;NtMaN>CMigr`ZPC0e~@VY#Q(UxJgJK2jRc{eTrcC;?9RKv~)!sQV6evwTaz z4UibC!7%cz58%Sr?`jUhp}uf99I>1v9YudY4iYZR?UkT5x5 zGg8eu5nKJDC5^Ij#M{-`qASzJO*EwTi#Zrl76DzjsM;%jti&dO4 z1EwtH{@}r1qD2UVvaX!vQrqFmEnpSyJs*5BfmO*#?)PE2cO2q`@#d)%$PvD7QHkDb z*#8qWd*}H-L9@0}G|;SgmgBOY-j1e5+)!Zn$_E0hfScdy%Yt4R#%zbXF0t(yF_tWckSX z`tS4n;31R%VS{-8#JMX7u?%cOM`#$qRsx5OVoYIE|VS`t7Fa2 zF(C-S>9w?c6GCUzKG|PnM(lwF*qPjwU4;+V1A6UTq5=A!4BShXQmbLPw;Q;}DOL{q zq;Pbja@SDofB+M5#R9@6s0z?fM8CJSawEB)+mCN~$}oF>fhsB5tNgaLSI8SA#LoWn%b?fDgR|s)&?96^rORWh5adK zcBFd&6uZC?$~mF6kDI{#J#W+(5#Rj^-pteb6}-8LPfWMB)yLm*`)`~Iw+1R{#SmpQ zEXit_&k9TGmtlNXqr-4tgC4lAuJOx5n#m+t3~70=fWtHxIB%EfC6_6*{)|8#=9THh zTb!+#=@HRFHt{eB`9Q#JSwT4GsG(<6TTUL(5S-#RpDx=P-)d+t5J zO|Q?XNiNkAE`Ege1yW(`mIa6T{%352xDa^6os$6gmk_jGIOrgyOu)Z^8JDJUkdFjxCyu$F#T$*NCMoGGo6$C{O|>Ge66DAw)+VrXH<}5 zsX+)a4FuU4m~6r5&2&KoahO!kX#KCVsc)kkxo9h7)C$D2t4NNo-7@#nAyq;Ccm7w; z7=9_eVF{_Kd!T6k<0zVd58fV2c# zLg$Sm4r6hga0o3$%S{c|BntodvL!+rtFq`eKf*yVJ+Gt%4P82zMVz75=nqR2#e-5} z+W25Ia1%JU+gIfiR4MktKW^{iJ0P%t_!U z5R>o|Xqr9;;&m?sj(ZGWjle1CgIbKTo&%qM8Rq+c%MagSk{|na>kbq8c7WLndc;1O z8=*PKKRmjzOKn754Xw=+vG(N367o}OiIkGA;n056&b5jBDQ&ja9Z#YnT^>iV3f#B? z%jGmuhbDg_gn_cG1%v;qrI22+&(r~xA;=80rTQbpuee}Dt14&cfFYzy#TKZOTnH;) zv3NV<@0GohqYtr2^RS`XC+kSNGe{E)}p%ntd z*%s=RLu^3x$jPE-bun5X5kGi{8O}9t1gAF1AoT}JFwDb3c`@3A^7U@7^(r{fz?Ct1 zWfkt;j<1qQkgcU0U0)6!NC28w;sWZiz@yCg@+rh3B)(e~{|T;m<~Ujs&35f!7gPH1 z9hW3-HA?9n-sp-Jbvva~|HJ12tzrvw;KaWw@XYR9aNw$+n>Tn>FMQpdqJ8IA?Lt?N z_eE*-)o~Nb{r=_16H>O4!}GdS#4v(epd^V7k>vG$i6WqZ{dq5hz3^ zFoF3E9{AHH*eehwLw{qY2cqaa7PGck3P_1jIidujK=-`gDmZ3WWT(f2JuYnVPugfq zGst!N@9bzP7m{~b2XprMVBlvwf-!7`<8?dN$dnM;DQ3ZGY^85UU~V*aTg>$nc2${c6MkURv{ZHt{W;IN)437pssC4o9XqBrGB7Ul2Sv*Hi4O&<}yTYCpFY-jH>CePsY&!F*+SMwR$ z6iE3SU6B%Xa)A4U{P*{3tUs=g%JJou+b^N91_)lcOvZK6K_1Gz)`iWVyH&7+VN~Oi#6jt8MKsyk0-^AY-nZ}vta`a!fGbpuHWbT^ z_0GC&VVO|Kh0j+HPZdA5iu_^5VbG5%>EX-M36b&y%-zlH1ShF$pk&)sC~CXI@EF?N zgZBYtK5$w^nthRkarYavoFc6HORWlSF3xP3zRhgUA8s=fRtMgZ?jKcui5SxD6p!1D zAz?it0nx$<&A=3ii_C&?R_KgkbO@LDeY9kKiUu#dHzuh5zlkDV2qZ7J>0O}|yqPrh zrrm>87?P^@A$D+Mplt_n-J5WmIi||t`Br`V0o^W7D~Ep{zT)-2Im1Wrc%qiZQb$A) ztv9#xIu9Xn839l3A;h<6no(GLi&0d2s$OsPNn;oq9kQ!vC>+n|2cEh(U%qQK(tInQ z8LzjHdnsKQ_S~{DZKh-rd3|R4(w{ngI#ayhsrzp}WoDNhiPFBVds{qsgTe>sY4LLq zA)dR?w`CjCO45P$_i7=HKS-;azpy_3_&l-6pn|KMXVO@-=KR`WeHqen+^FCsCvsoi zi#rl^_DkPpCG%gq$AB6BKiGTku&9!tZ*(Mx2~kNZNySVUMI}g7L|g$efn-obP_hUE z3_)ZOCF&YL$znn>NM;z2q+}K)GeeLJGvpbVbo+2vSCn1${hs?g_kQo=Upp)3obIl! zuCDN_Dx!R*6d~JvpKE<#(cE?xomqtjg+QZssxy1+WEQEP0Djv2Q z1nylyo1#PzNtBa&1x$AXAg^U|*fxDUIlt8h!t6zUM|k}K<1f{4+i%nwnM_O;Jc?on z964a$0c8Y-%&xPW;k>kM5fwM+08e^dAAndO4wT2e{S8rPeEF)|UswP^aDNm+ew!?t z-qTEmaL9M0HK`tCx2=VfE(|_e$iH9X)wKsLr2Cxwp7$Tw5oN9`KJ1q} z@`IgS0c4~o+QTr>2$2&1NGuQN*=hZ6T$xz@1Ty>e^vq!Z(V6$*_Z_y+07fqe(Lt#u zscp(#&(=;UyOB^ZMW3u-M2?Hg&%- zZi#z(_(FD}kcVkqAsUIy-`ESZm2F;yile#GP*N|8<^=Es-GrA}Wj!y-$6?)K;8w|LRZ*{*)6L>O)2txt=?*K-3O%>~Uh{7G1c z(fi(8w(s>>yjU=F^fqu=-?^)GE)HNa*!AOq8y8rX(ol?%O9a3VK-R{{%vouGN`SLA zpnZKTdVoAaJlTakhVo9qOPOB3wWf1y3ft_UI)c4f{=kH5w*j1aYPCUJR9AEK=Qa8R zXR5*Px#l*ISbp|EaSvcuKj6;n0Paj!1i6wzYJdnj3yYv4FWq-WUYG9?!-iLfr_alm zlj8*g^B$Zbhf6*TZJsT^uTjO~T&AnG9wkXFY57=JjV9X!9Q2aNN1v8671K@(tY zV&UkOK!;rG68?Lz-rZnJGf(|>Ap`puD4?OkS3j#nFMyM0`Uikwe-W-Et5bCH&qa{e zZ+!C_@(uI+owNA>@~J>b60aFJ^xPH|y9e-!7km_1`F`{Nc|*XzgaY@Nh5k1y9YW14 znC`8-aLCs)2w0Z@y)%F1b6f2c`ySWwjpQcKMp{3N1+xgca1Q8PIY2b@9o^!%(PANk zized=lmKoxB}Y5u;rs8g$lp$**FK0qD!&Nyi$K3PXz_{t|KSk>DM2K|)F=)wD;9b; zGK$bNF^qP)cHt4YG`IbqOQZv95IEPqHP8_R=ftrRfsg7(p$2;Zy}dg{p#DYHZ%bR zc-w$u=nO*C&f@g$x(6`6NVzx-EWr_;Y9prnmgYC{`S*{%UGrd0b?o5cvZ*@9%aREP zce5itWrIG2Odv`P6%5W4_$D~N|M9)Y`tKmzf%y4pbS7M|2cTS**9*y!u8X;BVJ3gO zw!x#hz+v1Rp*!?Ds0syQ@W3l{rWCT%Lg6ya7Q;(8aDkHXy@zKY2;2;)YjK=R=`bXt z<|ua)(C19xx|q?4s(+KIx>ddMt7;-5+e9-`q5L_8BF@84XpfCL* z#&-wd7cqVjkaMo`Ex)Sw9Ve&Pfwyd$9^w-)k|?Dt!MU;y@YR5p9Ua-9|hc7V8y4D z#SMdQeL%#55J&T?>q9<;RtqAwy5tKaV)xIxgDfUDTAVYHKxFUUAni5-*DwnKxobSK zDTl$^-^_yfAAhGhK*Zz*Qd28H_Yuo$Sn#&GfyXVBU=z#hnC=~b2JgD=*`MH6Cbb)a za@cksD!ib{ITEpA;85WV0NMWdd2jGV_A8*H5A>&RA+c?ounnhvHxYD!u2?8~jt+jS zpz782##LzWi_-$QMBM!wu3d zM7}v|xk-O)qmR=ROw4zjf59+x4DnEnm-$rmx-$S@0Rgq48*(0~-N-q>Mqgi=Q*45t zTTxJS-7AjyHbW6j5^~Tr>|affEgJ|OcxSnfa6I(zD}n8gwoR=&PC6|UWLWm_yeTW% z`^%$OYKLy=-j2(N+d()=s5<{jbff6?wDXObh7YWN%W>ijvOLm}dO-4&1lQF^9|E|p z@t;!DPioHA*ttG82OHUsbEwTBwNu(DnErBq&s_JRc83CdQ~#tJF{i&E%7F{2i@kWs z(H%?#L^EM747#XSeKsx>KnxozAKG>x89opBU&fDuKdf8*QU{ci&EWUaWtj_}7V!AQ zEmiK`WTcb*h|obLbb$H^S+e3&t2%X4tkr&Gzila3fpW2nxov)o4LWHc_V|a_Xrz)4 zY(R>%BkqEXaCesTqJWOmgKWZyZwgN##)v_sjzS8X!#LV`^L&YAk+ZW6o*srH+3pU= z!_tRGUqm%sD0};}A5gV z=b*IHXpnTa3|gv?UTcctpQUC>i<}gimJOZINkNj%c&U`XJrZ8@a}X;5c!ceI$m`(< z1kW}wG~z_wwzC%&Je`uhmq83t%!_L+a8qU6usM8$XW;HkP-$jN9%Y(9=M`2Q_Mr*t zGjtgQ&(z^l?KAZBIRq^QPnru}A9N2lyu%6asBeW<4vbj0H~|Vef6)@)d_39b;{_|7 zIY5~8Uzyy&6u5w)J`o=Z;(+lyr&3MUB9d||g7AC2JVJ@78NVg6kx%xuaI9VL;Y2@0 zwso&Ob~}aiXwrwV^^53kPNj;`%xZ-pIDtI&4TJDYW z<7mHZzI*J)FF-yhuM+Uy{UQOF2}MC^EW4s;Csj}(WnR-ea$}V;aCPM}?c=dpQ8?6y z|F~GzEU<101Qs?MYK0*kIv^_DIT0w}rKAP#ZoHm*4{(%g!(Y+M_`PUpC&Fbgm$k9) zh3Ah$3KZOWk|Mj;>on~C(th!ICBUw9u+DR&BXLs?@F4wkb{0SlIWFAur|KFnFxYJE|ck70Yu-5f9gUz8Vzpn@?miPxKoFXYGOTgDXWmh|f#w@)i&X zNp{$#5t>FF1cm7P{HOXiE!tAc3euwa=Lr(s_yaf%UY3$9AV8ZgDt$M^GJU}LB>Cp) zH8_$3ny2W-UqOSFd|d_0qop;BRuC3!uE`ApSSXN~2gSQ&G7kT359o2XLH=FUZKZd_ z5O1fK+>|=Vq;P8NNPy%^v<%>u>bO_WHiW`;@9i4jHA7{mW#-vT=^soAC$J@%>4a;*tG>@**M&|z8fHf=RIcO z8T&H(d` z%uCq$W4`Tw$QscYCdJuLnys=fhkh0!_ptbxFhie9ZWHUbF7Q7)tI=F7-6fykm>l2r zksYLqfSLkO6$qR6r;VWFb8lYE-Z=6P3wskg3(~Hh{bTL517$ zrFaO#cNu4reP`(e9RpIA_r_-%Oi0sjP~{J6ZT-)7 zD$>H{ZR*3Pz%75Q^e{K*ATTfLYjn86AQeRuR#8Z+M30#atoMus)#?ma1RL5rw4+9S zEZGP?6v^^x(r6Gtlah>g>MUq>A2&&V)6<+G+uhk$-z0u{%ISvyJEWIE+U`@X*2D9w zO$DnB6id9n5!nAwI957iog~oKp;BhRWa*qb+I$B%3NR0h`LkuN*!FEKq~+19-@h=Z z_=W{oJ)?4>uiwwYfme_?GwWH@-rnVeV97XWg@Mgt-tywz!#+2E=CwF9?)1 z99YqGQdB)>^F7#xJ-PpD2Kdv!T3Bb~1Av$STud@*PvLoxbI_5w7txG9JT}7DazPyKB@|JMQ#1_V*7&fS823}Rsu)K@!(7bwK zfhqKy-4jv)jfl6KIK;8o<;Fzb%V+>X63mwHszV$yA3b zP%G&xY(RZAb1dAwW{TM@n7hVWzugT_$hI31_oDG$^-!4G+ch1*L(9MavhT!C#!-qJ zV6t95SrCW?oF-)o5L684nXVuy2HM~XAfgi@pq)a}7&+N0aCJxh8i>g%3 z6U$^4uvVp%6P!Nr%@Xyxu~5P)k1YUnH+c@*CYFD$yQ{10MFqi#9UZb%0sXLs*}R|< zYoem1RZy@8-EjOqnoUu`n4Emq(B$J!+q55YQ|25kJ2_9<*WmxoNTsoQEx6VsGtgCV zMKH)%e!0u>!1vPpo3i>prVb<$e%{NiU@yO9%kGEw@-d+Ci=CssN;EhNdbo-Aq&kS( z!^S1IX#Vz}j}Q3lHZNekFkSV92T%jXrooORL(aDVV*;hbmxdLN0Wki8nJb7fdHf?u z;L+!kd{jnZJ7pnX!Gh)8P8dt_%8zd?Kt6NNOOnP+`E^TVw}P|42Ow(g8=`pU`N``E zAZ}7V%St5^&>a&BI*EuRGlQBIJ7L59(a|!4zYL*8b2ub#Xm z2^;;(!26E*f7$-uo#6jftg&)Mp$5gqde3d$OWK{gl3Omeq;7$b$nSQ7Q}gP^=Cxk$ z;p2PVa>xSseLyV?;NNMk>pLjMw2Q@K;!)J&=8D~mPVl^})8Hmktg$Vhgu~PAYFmSKQz-mjylaAX!V#UIaxuA&4RX=|l z{7Z9oA%`NlF_2gyxJp!~6H6UXM3I~u&51^1k)x0=w3^e7t+4OQ=YUm@7plnUv$uhh zZUUMOLkj2GpOnYHq>Nt5Tf+mW=^le3xv2axEBz@04BuP zZi%IxOkhdEUdIqACzT~XKV3zQ^~dA>c=r5S1PeXP+c>_#E)W#sVTH8=@e~s$PEFO8i)(O-s{<5G{;8@FmQXvK_6&V(+ z-b>;q{?>({8|4=K?hz>2mS-ERT`9$kK(+Xn+|IvG7bp=1Iad#yJ1B4$;dKp`^I;vz z*Orv?i#~T)ZT{61U3pVUmQ@L$))?T_>=)ffMHfA`)Mcz>vL0qXnKeDTzzjdGnb&Hk z1(ts^S^;JYD8d3f?H^#JXpPkEm?iI(Zx+Mo%tT1hv-yJ1uzyw~b zuqMcRc5mL_Y_s_B?mONS@e{5EDs>n&@I~Y5B zU-oQ4M}6S>8R72> z_sGA_^31|kw>&U`jb=B04H-#E>jzO7UV%Q!L1{SAiC3p+Pr`mw#K z&Le2=;&8>ekGPQ@Pqr{KLFRB^JDVH|USXs+JMse%{YTl70VhQyXvl8f3ySi>B3V0! zIUw;2Q%4;R6YGUFG9jkwpdZ_wLmY4oUa5eA!<8=<%;);Emm?koERkro=%`AH&MWHK zT@o!&BlsQj0jZ;bg%YfhL8eWFd)}$CL2e%U;aZ?1YcYpne$}V8pOd%0eX;NlR;V$@ zE}yh)yc~|(D`J|6yY<<%6Z+Tc7Bo}`f}G>SJcY(Z!zm9 z+O)$7~#*RhWZ_)q$NaL42^ zB82jYxnh}^7`BEyf4pg`wHC6(}bBe(hlHFEqR7l6l{$2(6Jj!?*fGvoG^&4R_za-u0Pw` ze+gumO!JV7v$2q71WL_8b}uVeh4?Zj68gk_*iPl-ZnzAlmkA&K7Wu1A|Gp3)x0KQs z3VnS!bWItg5WuHSWPeieKk8(Ef3pvc!iUEmFoLosL8=O*&Ec1q<}5=n1XCx<;{eDb z-!D3U?IqYTZ~hq~^X(}7^kW)KT*u<1k5S-{RR&Jtr1k( zh1BN}L5bzNphkmYl#tvK zSe-2U+tR0t{;bX7y(y@jX}q$HJ+VzqAxj_+cSz#%%_x^^5r(#u{0t{5?Ol6+0o^t2 zYW!?pPWPp0#Y^QSq{C61+6KFPLDhm%cvHn}4qVuy1k{1~GgtbhG7?xlPYXLM+w1-1 z8kjEZ>pQTyc#JiLuJStW@>Z{qSK1aISQBxjRW9OsHK-K{?WNQh|K~ZF3r_%T)kfRQ z2yW-k+HOdA-rt=Kyn^xCG433V(PF8+^m~bq?o^OeOFokdWnN1T`ui1ycu!8*-H*B2 zG+yNL$?oA3v`RSGCcHJuq8XY?sH1`d=L;Kya@6Jy~1P9b!J4cCocY(W1Ypi$j zkEVntRr%PHwvlAE6=o-2BP;h>Tk*Ne5v&Xyc^B8;eqimum4;_q*21P;_kurCrA!PFN+jcZn4n6ZcQVBWmgmLTl z-x%6z*z-;P6;%m??l}tB;0TX4b_*!>7;*x(v>b)Ny}!uP^731)&LJN$%MpAc z+})`}H>^>JHQZek-yCr-jdv4UgJ7<#`nk8Ss!EgEo306(Pd=ojGAI-|CTCrNOFIyu zv^zJcsb0mXn-vN{7G!){@Xjq5=FhFApOQ~l3AwUZ5(LCdJ^9^h$D0*qoIk$6`~223 z%-6%*{d2ZjB45o+o4${Hg{_3`4&|No3CoG>s2jA187k)0?=~CJB<3MR$t#X z4C6ZF8uIiN91drCn4(AY(N;F}5aO({oThBw9B&1NG*}cHpcWo1_k^^nGhB6FP(oMy zEE9UXG?^f1&aq8>_#w_f6(5#WLQl2|>=A0^9q!LepXz<@@M)8~?)%0*cYDQNnu)c> zjK$EaBQq6q5>g*iQ){IoR3~Qc94n3LZ`wv_VtGM%ORlz`y6-;TSfqGnwDL7u;?qW( zS137(&`ovyywgb`_bF)u=~{%GYn7CnR}=kkNBmwqaLJh*z22|lji03F81*OhMKDe# z?ctf~7<19_Oz7!)7p|liSG#|>$#MO3ZBLnUYH8D%a?h-^rrp71_00;K3L~hLP~HR) z6kQ@)!O_Cn>?2Md)2xUGdK(h*f9R)QyArqUN+X6D%<9R}9NXc@cN~!Fz2a}<= zdod|$GxeKh%f&gFHs@`AGI`H0W9{QgXZ7xPmD)Z%nrId~{=hE(lP-<|*nwvxm07az zZ)wxUc^?%GhEwZ4kFmC)O0tq_R$AN?ALyI<;>_R>C-nJj(hM-DGQwII?N}8Sml~OT zyg_qI@#APCT#s70r#AJKisXSKN1tE6Ot{W4V#b$o*uOm~r7}Vr?E$ea6rFZ9e{ihE zSuW3`x5GWrf<50eh0di&h#NS=+HwT_deSR=+C@5Arw`+~flFV=xMsGzHN>q!h`XKq zRFBN!TVV0r)#gcqv5VzZhWRk&z^#CcZH6t;TF*?Ah@RBY;{H^beqqxUcf;)iBluP$ z#PiwHW%=i>ILrxMd_4G&ZkTpgNJ`;Wz%;4*4|iQ>aT%&b5%`ha7!^eGE-$N{@Bl!^ z{22o+tv`jBFLry6FuxmyJxX#80mi zdfWMd+U!x`Ln*_KA;(p&cd=%4A#G$$d7O1m_;V42`;ot?W(wpm*L*F|#cn%d#Gsx% zWt-PS95j3{>bPC9jYGxwPY%O%HJ)7OGn02u&y9uTZ9Lc|?!k#H9GZQVIq|_sVILwC z)1cQMtukzTA1PE;E@zDin<1{fkP@52TBcm#eIzNb@(h;xDXd!!T}w#vq*Nqn6|V@6 z$#{w8_3OG_`7&~Z+SVMR&|AaicgKD9@`tL9J10f`v2In^Fq3Pvh&*v4>HQBJ%tIb3 z&J0)^-Xg_YbvWS7%W;zxtceu@>FvB_IjB>WwDT_c=BoV{yYsdEtqr!JQBefJ^r*EJ zGrcc8_zL`Oa|)^$cd%CncJP_(1AVTX3Ar;K(MT!Wtz9!Ieg&di-tTV;=~bxZs4@3t zUfDvT-*n%H(0LIrng6M|WBe$(YV~yOpTaG*siraUT2p2q&NEC&JCLv6x3!Fn9x8F5 zXQ*oOQddkg>%3+~utjOyWg0UL&WUtO7!i^SBq`&D9U9}*lY1G205-B8^X;A}mkN)_ z=e=`ZN`^I6E@M+WuX>H&z4UJ7Nc8878LOxeO89xgAdKVYD+<-jcWXO}lOi>i6p_u-6$1WqSCbT#F=GZRB7E>OUs4~~rCA_u8@(*Yox@#}2 z(tAbM+Ojq)A<=6WdMD{ztqbbhK2C%v{k-EX+WwZ1T}-!yaXv)0xwKSw-Tf9?nMgH{ zvV9J0g_p-Z6mTKbJ3?A&zYh2JwU}l9g$3~0g?>5UCep?kKTrw4nm32d94Hb}x3!(w za>5n=e0HnA(AntXS6#c&Yh=Bu=J7O21(o=rwoZoV#oCm)hS-Vav8cM6;D<9VuRWHA z^%SOsuI_zGk<+5$jQZxjw2rf7RJw>xI$PAup6JRo9MVG{LmWV}-+W0@i>!@&vgH{G z_t`cz2>5_YVO>K6$8s0^pFC{0O%Jfff-2fPURxNg$x|xtC5-(X%Vb=OOecg9lXuQW zxU1(V$#X@As+473VA*X{x0SKBzaDv&5_~NHHYm98D^d(BuHz7$78d&{I-P$QyG>D$ zj2bpRcEr%E9!;<>F*t{}4xL_sVJ@%xy{X`LPDFwhUzK8Wm21;zM3L!jS<51+o(|0s z|ExYA`9sEO)2@3Yil{{&{TiRFM!l*EyBIg=HJ-|foizhRcyJtUm$B`*u=k{0q+Zcu@p10D6O(Sw5GHX2 zRyK7v9BJA!AH7NG^`w5~2G^@wRuo<@4=1;v2WlL_+lhqADw(QK%A8R9Q$@VEgVE=< z-35Qn#=h?Ie!A*-@8?@z!$l~>3v_8@nV_xvixkR9y2Xudp>!@|<%+vxKO!Pj;ABf- zQ(2O+W3b+>(JLP^umrgm4s=Fi+S)U+nflJ46THl;ud`Nr5Y0nJk_4=;Jq3jWU^tGn zr+fY%z-+Xfw?~xN!TxfxzQ$d)CaclY)4dYCKFXmi2Qg0a`5A`Y&n z^eVhgWOTDhx<;Tu6ovRa8F$phzI@-B6Wj);qWGN!w`#{BOzMS=R#KiF9KlF)qxXwFpxt~! z!sZ39pqKl0=VM(_4F4z8(Yr#c+dWa$pm|=q^$rJA;fM0`{ngt)2_Vvs-IuWRqzf0F z4k^0p!}O>&?k*)w=xuF|H(>zWhOb;|_EoNYetKnM8No^(cS*&#eGTF@b=UJvo43kb z5?iBUUMAa%aiaxmSNtAZ0Un&`{fsaqFA@@;N@9BKdPk|JZPyo;wY6~;`bZ8YU^U{G)5j$TGVo~{$%-TnlpWcz7reqqqo z0d!e{U;yuK90f(g0v94exjyPaAlS6~uG&iD0**TCz)t_ZnGnhD)cfSIBGVsHqZzzA%Q#FOc17ipVX9aI#9x zdnE1rQ5kKoLh{xxiGxy^WppW`|DN1nhQQpF>tdVbXWeH@>^>&r)jINYZedfl7WA|P z09_@QV$+9-d5cUR9wzH4N;5XM2k!Q@?nYPE8JbMBUty^TN6uga_PY{m=yFmQrJXqp zhC8UFmJ!nya7kaSyKw`NG7$cPK^fRs=s%dV3H^-B>e`1%HV9Tfnk;+pR>;Sj?z&vz zV%%KS^coFz#8Y?0y?mi!=MZ6E*CZq$9hi|Vj&aiJ60IS-6l*mx%qA@bR@ds@9mVdr zkQ8{;^-X)L)=i9oit!nCL}m{1jp3fkge|XXvMM8aHJp$h5+~|~$m{4PH$1HhE*tmN z+<8=d*K+&7zNn1yWZh}y8d|LdMFNGbG-j-A?2?_N7rMB8*-LS`K^VT|J!U);T-&7aP$Ciz zrVZnEm4Qkc4+}eAG=7W~TWRc(B!4uXH1wgakygx`eA?r3olsq?!2L3M>n-YWT{?dm za%U2&%}vG~Eu@u>Nvo+6fq#v4fcrDWIcdMH@}ohXEe{%)pZUM%%Vjo6jz^=S2jq_0 zWgFitlgBaq*Mt3MC-uE^I=h?yk*;R?il_DE=Z&|79d+2Z1z|w!8R1=RhQ6U?7u4p7 z8ckF#M#)uUI4z36k-&NTHojNCH&plXfkPZ}f10txI<_lmo`>sNGM(9l6#751-(|-J zbG`r6%xh?Qjq{GTo{f!H(QNBf`N96r^2yjTocW|{mXP)q`X9H(RLs^jRl7#7y2I*~adR@7E1wkTHm>_x^`^q`I{D+)BjJvE#(fdY!IP=Z2;_jCNA3$y+N;tOHV&Xk+G&3489< z#F}BTiUnwA@}cp;W`>lppNMR*)F>m1ULM|aIX8dI$VvPHu|%pkBZ2VXW~Q;euW!w7 zuLV(8OZSsp_7j^gBpdfuBW*LP2E+1QbS)@MyF6;kh|$t*vz;G_KsC_IFS~vmgzX-~ zd?9*nRT%I!n}9N6kT+S6n+iJfWbkax)yb0#elgbG%9~6RE2np-kMORfHF4mdSfHgQ z{-oJ{tpUv`W4%hJ{qJ{ZAn|yN<(~yz{&zFP&z7!?YC>8nSIw&RrH;zB<;xbh;AS?@ z#($2%ZOuDmvM0KUW_!3fbsW)I=hOOP!;P+X9zJjja1T%F-aT~Wt&~pe#2l$)?qS;S zbtjC#mImjK*GG?2>zraOdqp3mSs(T4R#gwA_j=%*l#O4K*xGd!@w7|*r873tX6AA@ zb{TWZF3z)ir&Ux=UIjh@Qiq%T)rj=F$yhvz`26TNZ5QWfLAPVh{dJULbII}ozB24K z^-Gm^h4$HMRA44mJ&b1vS7oTiebiB5iYbFcDlK@Zs&6*)#U+_uyrkEYpHbvrF|AphO)L3F-X3Pqk(36!BeW?gOd>* zvQLQq(|Me|Bk?K7L-LU_!D4Q|8`Zd!&dE{wTTM|p&u2UiOr2_s3l3J(!t@E6?{a_U z!ZDhlCL~MCtly>);-+W(lpz;2TA5x>s4;qt?C{xJZTrDlEG;Ursr+D~>NOqYBT_F} zh#aVRBsv19md=v!TEMyUOKeJ+8@p=pNYw1efqBWqta{7$RX=U8m5}TeE(SU zQKI_|a*A;O`IcIXoc@vP+G!O+W)=M{k(M(KAJ#=(I2}=&=@1z2X1n>{#Slk3#14a9CG+pAFHX791|W{?<8HpJX~^J#I_g=i)C9UN4XKfAXMUFKMbJGCcztxLTlwG=y+xtu??#+o1wdOvc#gBgUlFE%r_=wQ z2-Cl=6N*huFGR&P6*un#8LkUa@$41rmQJBubQ&JO?|z2DjEjQ0i=O%`1pNI}ly<;z zgcGH=Kz-fkV)OAoD<_U;%Y%ICMB!JjO$W4q03i4fgyQ$}8%|kvRD_od9PRAi8xR77 z%QlhuaX+K}!<%z(46pHxPync-y#_wAa9;P~1G+@O_#^!84Jgbg3aSM_e%jl|TUdWQ zpNpfxkR16xfsrkI#o{R}jniGrtibjFJQ}1|sFXT_#tO@cs6u{NxH)i@sexx7z-Z&4 z<-?@&IqqM71L+S9947&a1LVI$`MBIR2<7FmP^+%OyORMi!J_!dvV{7DRQrWg`q$to#UIO(=2UefDRvv(A_b)>zCAdu5 z*?H*lFM!TJ3g!Pr0G&nWWT^%JQlAjN)TbXs$1nBi7tsGl$?*&5|NkA(|3_u%$Grh) zzhCOpFZJms^=}D$bHS|e%D=Dx|L0g|dVR15mwA-CQ$D#AnUwPTe@RbTMIE|?^?MWP zx*ta_lM?4vM%cyWJApcH`Yh*J5T0`lQDq{=vlY&zB1ZK*+LQkwG3@jCPSDfVKZ-%^ zA8&0caEj~UMHuW)2K^E1c)Yk)6dKbfLdHS=m6CX4+K}^XrLiE=<&#?L2g}H1Rb|l3 zZWU-Nuy)+vn7YGbCOxXu(rEIz^Prk>?q~b82=OOPP;GjDB|n5oc^W6v$jjm=Px>r* zI@z;>$LK%J{dlJe()C>ah#7KNq1V2p3%mXv!bGg~!!}_3k3By82OfHd>n7wh?f zKL+&H>#R^SLHq|&HiGUZnMq%FgD%(-ptw~?%#En!-&(=+2s}Ftn&<&n%0kV)H$^W=Hn@s1hx<-=L2wy#ndv zB8GzApwz!O6KI}*rS>9w`V|>2a-dz$_^>e}S6r^_=i(keEyBX5(BEk4g>y@#5l^yP zuSm};r2F8X{i2BMTUXdXwG4KqeN49_#_)$~rBTnEqT`?8l3|bvIl} zBr{3En2F`-4mg|g^x()6jW>Vf$P_Mk&aWQVoB^^a3|_Cbp;^JQbN=VTn~SLO@rHU6oDFZF1^ZclfMC{$LaK} zJa`Law!_616V6})muQXs~b>UTvn=?fP`fz;#-;Yf(BHiHNfdO z)6IOlwk!^b-{46e?7`F-TlrOo6zns?ucpw*q*^ws(8AqVQfn*5N{aFNs4X z+|#t3G97VHhRJ#VdJM@gdoWcab(mmV9;xQ*-f2|=idpbU=EHmAIUsOJ6uGym_Rhm=3yD)2g2>OJz+5SUj+qhRF z$|gC=#B8zqZ4}e@&A3g+G*36K?(rBoYcP%r>%JJ5Vq&=Fa`?Z`1SkhC@Sau=^Bvlu zWCCLPJR?4=2;UFaWEoR!_B}-PJn#0&@lLO-axG&y&Qw~E^HjRN>DYnJM{J(LnV#ZT zYA<dGM*uN})PaT@dB{owm@<#5*DmHWTvU`v}Jd>UcM0ZCjQvAt{-fw!; zNe4ecCnP>1wFf1YEVEA1m_#l@wd3RXtMTsOCU zsn>TljWVV1uX2&h_ptt^6xdJnb8Og=n{| zK0O}&TBLS`(qHox|7z@6Pb#U~vRg2wggZfD{c_5N7F-l0;l^|zsQ9D@MFM!a5#giu z=`;!ISR{E29AoCxBR1}xLCsFu#D4F>wi?a$&B3%!8smk}9J*iNW%d}+d&&X+)_q|U8&r|Q7&-Fc!Onq$uYzj zDzh}xQ<|sT!}XkjQ%z-0Ib)pHzL#Nt*eOzIoGJ`xAe6uBjMGXd8zNrVW22_W8_rWB zRMfeV*`bcq;Y183a&E%clwnMl79_GzR+`>ROeJcfrUZ*9pYM}DJ6UjG8mcNQ)6gfFVd***qJ)b+}8cINicvh^ruwmu;=(yIS=}qo&wK2GtmRme|}ZW z-;?A~j+0KkeyN={D*upj72huu5Qe*{#qTH56XeTL(%3XbI?IJ{BV6RqB@AT`CZjW& z+qcxJHEKv=Cd;JV+G?_+6@8C=wC#`bKyt1V_?=m{yp0DXcgxah)LdLv(PbuCk(<}! z>nVd25-yt*`$)*dAl}%@d)Y7y9}HY;SMth-)GKC49=>3@{f3aeGbwvky@U`u?j4-m zF>G`CU=y=RaaT%L-hlO7p;tEL!lkl4dPGl6c=oQOoVgk4Exe_H#zdc~T!y^E4uxq# ze9y;c8mXhN^LjjIG1Ibx@)Vk>ePzLL&qv{~D_;q}dm~Kxr-?EfNO&i+@P?c)+}e%x zvsg>ik^F5=I2UOfxBjz7M}7A6kg>DH*?oiy8-=ok1#`_T6KIxQfPVwfNd-oSNM?+EhMHC#y2wiI6=}uq$cq%KdU)eTi?{(m(uA z@`oek`k&9f2A#(zAkDykH6rqU4_f@bWdTp>^ixVt5Jv#+S+dT6IzOd(mpx5tM<#CO zY`eV*@^}`HbKCnv1bSnj{a{_k*ii=kYm}it2;O-|cPy%$G`Btdf^YIY5qd{lwh5_T zdT!jmkvP$$=-hwCqfOCSExh&SHiDv?*)7#<9)#$sXYx1qDp0WA0J489`%pjeeyni^H?LXLAJlel`v%!0KV@_Adwf_AqF%Orb&q^jnX++n zrr=@G7FWv^*s9p4Xr!lP5ToA5zPf(lcInb*=>uKbWw|cplU_?Z(vzfJYa<`Zayy|& z)KA$o@3x+&<(Va3wgN#SwWF;2@=%+r`Q^(UA$`$yxiSi=k zW^?fOM%jeL^9JeJ!=wr2KP-=x#T;laG;P>J2=q;Q)^je~5+zY3*PV6fQ<3nULtXg< zssqz)O?r!O&jg>)$KR(V{-os{v!Q%YwjV|xjj5?IWJU(W6AR{5AEUh~oKqvNBln7p zvC;(-xZxhMU7`Jk!~APpp1!*u_zx`hP4`wRhP6<*zMIlkaLk>wm~OkE5d7{tM^a`1 z@uiNUeA~8WDcGr6p*0xm9a8s68qd9UcJCj@$S`PRMTHG-YEJ-R$#JDge-8`UuY%H^ zK0cyIl|B>YvEz&<$&1{GIs@*w$bfqiD+;GmdRoUFf(~Vi;Kq8K*;W)@j@6YNTew{U z+OMak2Oo>Jtu!YF7q=aXDFJ0PI#csEhT+_F^9QF{W69$^f+%{cN z7$YnBG*xg7B1zt6_FeJ#0JdXO@{pp!EHMvtRPK$ZZ=ptNKT^=oka5*cwOqiST(4RF z^wJZVQ0rxPRu+~bZS`u`e!C4SjX+Xl4MKqt5!Nk+_1wFIcrH!l3~fU*v$yQ*G_wV` zcazW_8_*si#L{PoPLVSuZ^ZMD29*Q^PZ)v((!TW`-pUH#aN$$sZCqZW%JWcGEt_}^ zQe-57r_)M-7~VY9KJGDQw;HgH>(uCR;FT%gY3%MYB7bmdGfveeSWN*4mlh9Ixw)3a zgHO}>)*{vm_VT6Do$+Z{7VjPsVFFczQh+Y2u+;u|4o6}c`D;_s{YnylaAH=w5XxZg zRIMZVVWAm^>9SCzI;f2?a8cZqdN>H0%0tjk%v*$s9K)o2VkW+O=1|y`@(5Dt8@$YD zrgd%kr&{aaQ8rDLt4t6X%%nPmP&A{y)b|2&$LwqyBC|;`hW16R5hId`tQDuVEKY;; zp3SY`Iyn-AOA{+QIWaHmBV;+Au11_(Cs?d6oqSJvx>TgwkNHja9YVyo_`n&{gFw$d zL^9$GF+k;GiPhts)waR$gnP}7q%2d-{+zrPMj6xX;o!cGsqxtHIzk%Cb!0S~l-*Ts zT6+mAz$@UXW-whEBTHo`oOS-v#fZXCf-#8+L{`SJ_|V@(pK;}%Tt6yAkG|qCERBgd zW&ScG+*Gxw2BY`efg5{UpSnIdF5! zl}GztX%xM1+|>0T^@E1@ZmBz3yDf>B86#{{E1qVVm|@)C_u8Wcojfg<+}~1bG^|UU zPRJ&{zy>s7C_#Vr_{nC(NiF9X^Ln_}N*F5AJ8|YE=)hp5YQo1n)5>&v<9JDwP@o#U z<4yPdbR5?n1(zMTTW@iLsd>R=`bqSQavOO|lNr6`-HD26&bC{f{o8wP@FpwH2ze?P zp*s4z6XNLy=nct68-BYb#$FRKz1DLxV-T;&&?n$`j-ccs$#MSO9a0?|k~OvYtkxmK z1yPi_Ff3P=k*ju`#C4e$riigYN^m)YoEcog*>A|{N&Q4BvGntyCX>no6=|8-qxan& z*c+}WB)_RG!!n4;eu}5HuT#IqcWWBYW@aZ|FD>&!bW+cKFvp0{$B@bXYw2M-DnE~s z8KlAudUM8eb#V>Mrz->Gt>ux0&Z^-R~4#cGTpOpG({Kpu6w9kH%B9<;KQi z?Z2=o@@j@p&ybtVZSMKg+eWm7{My8G4%@gn&*9VJ`OB{6^_OdTN}w5`h#y(nkbSlOM?U$A#_ zwy6ZIFb;IdHxp6rbIFREF46xSCm&DqpE`u6TyvYId*1uxncMk1dzu5snf#l#o7jp% z>FGG>xxU!(xZfVu-fyCQmKnc8J|j01C3LU6JHdGrjo;YcftIsDlQryzL(Y^8_xpFq zZxE7Uh}l!`-%B?dZ*`va7r$rx+e}yj88zf!Py3=MgrYiL3)3(&utXfVQg8?Va$`k2 zvadk*36N^!WR4P5oVyl3*7L zk#Q*3cA@p{!Nfeh<=pdL;!JH&=Lo0ktBFu)9e;?$r`ZtZPv#m;$8#Q`O3W3emD&|o zSM7?yHkDX1zF2l^*g9KMs~BTx)Qa0%{7Z#xiT-$LQgRQEYcPE6Ds;J@w zi5~bi-uN1BcE(@~d4yrF+CFaXDYKSu_lk_DpeXnLyzWs|BT2(qB&~@Dp^hV!(Co{7 z8QTcbvpLtZ!Jjyj8F5j8mAL72`}*K;K@ww5q{o`Ey*m+0GNH~rENzVPAO~gBJ@drq z1Idq`^9q_%FnhY`inHG22E|zeYVXc)XYKrv?x_1&^!B!%#8QbQS;XdNT4dZ!OF8F; z;mO^)QMm|{iO8$mC1~P3Sq1u=^rPy|ANZU7s13IYs>DWVdqbjpKm1 zy`@T2T2{^u2bav+DhO(p#CPXv;suCDYd0*;rdUcH;({6ZGpJm&EoG&b^}1$oMJJtQ zs36AF?1x`XnoVmrzX)D#&w#2`2F?Ifxlnq}PY-;`6#5hsX!v9(Qyk3vbvb*O(s9n?uTX@_2>Y)0^-;@>R~cvtG|gqd70-`x$$3wD6X z1v&VZ+NJNE!zR-3tLxt=?gj9Ed7gP%HTSHK{x7)DJMgS?p*c2g17ZsR$l2M=IF}n0 zLYYx7r;Skjl3l%}jj3qp~GgP#>fz4G=}O6?-AK z+)IGauAsO!#v8I<`oIT{GXIB>q``y5*F|;#4^Lqk2*OI8v10oVBV~XvbW-4-cL7BY zK(+6?vSPM=&udFZekj5q_^fE1_!^90!3{b@?V4D*EHQ2?*@ECTx`AsAR9CJ8R4*qK zu!E}X%Py`?3s5fpF$u^56AthRQuITuhs(aUIS>TnEuNoO0ZU8C1;2;xX8ygQ|2S;( zQ}Af!%TcI_F!P2D2!18ueM@(kU&Qk!fUVw&K$I&qfFkTaC~Zsa#5z?Fm6x`D1t80y z+AL4BV0BwP|_&V95nN#GIZ?D1dp5c!8w}9!}f8{1p-lnvL;mTzIz% zEKQpiRNC*T<5@)IrODFIKNacJFAd$f3| zUR|KC+xj<&k0B!RfeE%trwS~7WJzXj1k8~wd72LtOD!A&XVVvd_r~(X9|;D8qrcC1 z4hR_`=qwY zvagvO2K0`-ewrKTX@x`JcRXJ!+&klMhFxN%Ci>vfwm&ihd!-%AG4-=%ve_W&9aozjcdtZD1b|FYAfm6c7 zX?~+6QNS{gGaVq8lWNfNT$zpQ&FXltOH*;r)NY6la$cmVEg*;RM{z6smR>N=RpH^#LGB zsIuuvG88fG1sAMS{m@X@=6{=Q`M;A_w-_aH86=$8Ixxi%PpKC~MjQ10ruOfJ<{#8P zLQD_3lbf0Y{*BEal}7I0Dz&O2wY%1V+g*agA?jdVzUBBSDLysX-8Ij>)g-#-Bnhtz zX7@VXlH($peu!$~GqHOcW!#4&65RW(zfxUXo-b}yyAdzsJ4!=kNOOyL*}oN?e{Ie# zV99v}$#cZaliS<{nb3%)x-@gE&+zK)@sd{N?M06|b}O(qGRHDVp3Ax+K(O@L?0+UF z+yji{_l|frtA3sP2T}g*4kpI|btuNSMC^Ku2nRfT^4-RkJ-NTn;8qqXI@^MC(vUwr zjA+pUHkN4vN}-qchZ`vwYa^~LF4BW|pPT~V=5P%kf+!hE+QC?bb<`87 zXj(>w+Gm9CWwT<&9pS>myFxVfxH33>K1|^YAE&PelZigEYp-$NPt}mnD>?qrB%`n0 z+yqqaZ!Cv7FFDqphCS*#DTv#9*vAXsN2A%XON_TbqZW&jYT^j!-%_rVs9#SKFDcLl zAMorBxxYue5%tjS0^HNBdOSOhmJAtqtf#-9fygA|qd{EZtF5xKWfLIhOWAp&&kFNS zK1j)j2{Lry?j>a)&*y^O9Tu$#2H9?=SIFh5A+=ABm+CwY;t~n7{`*@~&dqcAA!}VB z@P%RVJ=tB2A2OO{?@d6={$b^f(VBhl%G}ub@!|5VV$3-0{zLsx#XG_>m-LK;`+CBh zoe9(U{19{LkDI;^D~m>dNY}+#*O?es!;O>l`ren2k6`P$s&`ICF7WKPd-Lr3oZze$ zcOcBVIS)xs^vroTNpB7wlihkMD^5qoJ6fliu7-}o5+=VzfH8@aRaV)*r6Qmn9Iza| zXVOLmuwNMU5gtgKc4c>3YApu5=~al{h~SxhkGTszrDUk9=3+j+PC?KJdMTp3kd3}6 z?~J(5pwo$?b})+N5>B`*&6TuurG2%}VKYcx{^PadOD_2QS*eB#{(S0*@i`kYIXf|+ z;PVr_VTuwG+hHzo{{0ry_i}@smA0|dA{wBg6gwb!P$!?iN6csED*67B{h zT$c*M3>BCI9-#d+8l9?ecuKj0LoNQb8!~Lg#=jWDtX>qp6kcqrO=C!Ldd^4V1UaF* ztd?2F(D{~`&M|1PUhKQfzjVTXLoHJ-qN(#Ps*7kN8Zb@EL(8}_b*`v5yqI3I!wFY> z4D-jTA2kgrZn7{nbPCKWqF1%7CkQg#!=Xbk2gGurEze2ZCzrVA=y6<&2nFO=x4Ypv zN@K$2=+$VzyI-~lTCuX!y#Ny>e$V(RAcK;_Z8eL?u}C4|XfWf+VdL4*kEIZ+h!!Ir zC>B3YrVSZV>}Tm*8PtN7Ba;Z`@gQUNWex?%Achq>Gs}^o7T`h)q>SBqFqKWb#a)XZ z?9sf6DLonLkq^Z(T@*nnqTO#k0XHJV6muN3COu3$?Cg$C@hOftzW44~>H+_b)+rtF z!Ch|u$Ym@LF>1> zh~@}4#CFL@aY-RPf+Bnji}p8_Dfeh&h9VV?)QxA0Sw2B@>=JhEEIwMFTCObj*!VhX zH|=yn@gl_`E4Yb=dYJ;W-*$0J9BVSVw6d4=>@}q& zL#+!r^Jzty7?%4{0cg0KP;(0#Xd};iQEZ^MS z8C?k>AK6D$#i=z-o*`XeR+yoe2YFx*SF%^53xP|74)#i@<%*A)YxX zeM)zB$PmmNwvz|a&io+GEm`ekNzhmwqWg+c=EaKiE zZ4ILL77E7sIo%Aac$WpV(V=cj5`MYq5>d?Z8nlQV`s2XiHkh--h7^Rv1GVo{ky!Hd zQ*e%Q>0dt%L|OO@SU!WuWMI}AsQk2U9ElVKM9neqsw!Tc2jCrc+NPRhC`UMc0Qb{A zkz6Qg)<$)UT_Bnl^_zA3yF|Gy|GwKR{@-)&FCp%w0?Y7wWZ?!p2NVmXC^O_c`NnE& zCOs+fFARY_PmXbnXc3KXx5D?F_}~2?xubkw_oS^4p^1B{O97t85fb73Z>@^|$pt@u zbn^h4T~Kj=7zjW-2bMxO_(5{@;o#77k6#D?SNd=lghL}ewHu}4`1#^-N+HjW!{wPo z?1PlzJdRlMosz_DjYnXVO?KKXWVF4Pdg}JpJsXsbi`qr|WqkWVzV)izKwR}tYp04; zR{>hiOiv8vOY*%8%;}|t+8auLu-xf)D3>Jsp!jOjmGRIkExMVm0H48a3xhS*^AJ~G zezWsu^X1>KxXlFY*&Z_N6%iSj1N1iFD@YP3>^*!;?yQfTyN=fMxrf8Vozs*bgk18 zKjuH)DKLEhzwl0hy5~cZ#&Y~o{eU@oZd=O}8MqcO@bZ8LEh(;t5W4^w7)(NHYXYt+ z0nChK^!fv2Fx0}2HX<<2JrwlGNXx|_jAY_MfxS=iJxyruU-v{Tw(d7DI(l4g( ztwmXV1v`(ZgI8KBW4SlsUxRRBYrrPw+k zyL^D(9vCVA1t|cIs-z%ODN(zVc#hqj5ij82DjBdqhR<7rfc_8`1p33dH2n|YgW-{!i8W`H_`VJ@x+=Xa)`(wGT`^RUOyv=HvzII|HD^r(*wTol_)XN&?1F4Ac*0c zwu8i-I|ZzjPS)S=;m`dUAle#{ zHEg2{UKR#)fy3*bdJ+a7kqxc^wB+Q^A#?x0G)e9BKXr{OApQJ>2|tqLfDm0k*XW3q z{6c8KAv#O_2RAK%3msbf!K{x{!XS!*`Yu*TO`1ET0fIb#*X$hVa7Pg+2&83=+!+t};Mhlv=2+RV*j zCnexmB3>S5a{(?DcSg9(wI{bZ6q(o1FR%Vc)RC7RUOk$;L(zTFR7LMfMl-`H4MyRX zX(nI{-$3;i9o8B+3)IB;7l(dL$VqGwOx=PIcd6I-p)GSlUw*YA`Geq^#@0fCC9xKLl#mx%S4lE;M@ zuuUcsic@fvK6Dtry7hYTRdzUwf>15rQAbxp{+{h+)lKwsvBbVUf+K86%h^i>PW5Hw_Zr{dN!u0w|BC&XbG@*%zkXP6cp?YSGL#Gv=uG09p72TMiqZ5<0Q0^Wx?pnG5n;Sc)jqhni^ET$}w?nkctDCGHcsc9rAfcAc} z1I(?jqKmLJpM@D(Pcj)_@>(ct5A`x~5u&m#z0ue06&}lkAv9b+k-DG60L{bM&z~kL zZ|=4@XynJ(Y+-r2u%5JZte44P#iv_IUF>FEk#mbYcNX)uCdKNm?fcW4dPpvvZF5At zJzUVxFdZXy!c)np+pnl^TAICsn{_KrRq3Mh-Nzkda%PKpwITy}Bco0ycQXDJM4d&% z715JWCtpu5pDphJKKs2Wo+n9jN1+p3J-uHK48DfxIg~VbjNLbU8NZ|&9`b4q-+?l@ zJPBKAdwu|W?t#kCq=$J@xLcVs&ez?!8niC-2363j!Og>WDrRqeO!cw6tTxDmL-08C zXJ9cOKVPeVooe84mg^B+aZb&9X=*6i!S~I{#@fdz52{bT!R|-SRUDSKt~=n+C|}sEiz{R%blQ&vEJGV|?9u@_vTl2(xGRmsx zN#|1va@cvyp}cri*@SJ)F+|wtgpeA>VRGB!oK5t|g8TO<8#isck$TfSpe0?hZI$p7~vCV*%%|c zjxUCO-ZYjmxK_Ts*u4YURn&?4QN@w_Z&&rk)C`E;NHTOuGMxQn{kd0yt4xO8x#MB# z>~^7v_p!$6EmeC*UqcsuI!BkCiJa!kbpv{l^UgPNShg;uIpF6C)69lG#p>%zS2_(R z8eMu^YBq#4o6ng7DZ8Hv9Y%X=7JE-`Vr{ka7AlKwZ?k3iB9@*yiyM2*_nNy+W?JuF z^499(Y|ZJHe)+9bbI-$BfAO<(hfa8N(q0=<$8ohTn#Z7=2%DDX+Ff58Uw65jQ>uOUi1ETgBpLmrUhrXb`)|r5`^eeOpv< z6J`xki*_n^U{jIoAC)m}S{6RlCFG_sLQ^Z1iq1-vwM-sQ)g#!-jiOe%+Z@VCgnP#C zGQ^`rlkjdcS0-c};%7QGVe3V%y9t8(4wLw~_e&>()h!c02rqS%_FT=HEtoNj6*N=s zP(8|g`oo$cuV%)eZteNc=KsnC(71-IE?;EheXO4NHr!{TFm;B_a??y1PkTly*TGJ2 z7e9y<$XQMhG`ft+o^DV03>Z>s=O(@n5#3>zk*I5bMoGm@PZ?Ao08Tx=gJ>8fWwet?I#MW zduZ(p)Vp!++q@BN@v#~@%os`#-gCP-fY8T%c8V&j@A+o?$xGJeu&qW60)hv)Bo9p0{;RiyOML6s-5nU|m^;eUkU)B2T`ntj!$_A0VJ( zCku;l45(CQ4a;%;2dNESFT)~?3>>SMI)wNl428HNUxWO=C&Sq?X&n_^qIF7h;@tw_ zn1DJOL?mHraG8B~tZ&!TmW(m@a@t=zYE$ZJ3k7iW@Ei?9;kXYoZU1)6SEtlSmR4$%@j9$xuGPJXPTwAQk65Hgr zlT%uM>4py-L$lwp(fMw$^4@KTU~ciqo-@rlTc=}g+qpm)65e!;={B1ltId{57Yj_~ zKbS5S%mY_%Wo8{6pAR!{n45S|v-Nc%T3D`eE5UC6)OEA*rtz51*n*!?CC$)KbB$fx2ySywO7oIsx%U687 zcsz7I^48-X$8UJS{2%==doSFF>MYyVNp!h4YY(Z>`NDp9huCo?PYSNA2|Y*zBe>oIozoIgMIPhg#fQE$Os; zD@@KLz2-!N+oZXyN15Jd^z&_VH}40@D}Plyc=FCo#nCjE+N+s-u&=O;4Kg&7@9Wg2 zuw)&}-iX4>e27#LQ`LU1_m_!Pq`<1v{)Zm5#mSahtHN!Ey4^0V{8)-PMrbfNCE+Ud zRn>!*#&My1Xy}&v`>SGtZq>0b4N^~ODvhV8Wt*sx`=jA~TCTkBlHrB@vv7%iL5gnY zoPjkTBk4WM2>5`5yP|f<7Fus5#nrHHO;Y2Utn;>cY4taFMf#D8oYvA_j?Rl`R=>`D zTSB)-K#2~i-4-i0p9@uu5~wlL>tv{6{s_!l?RCNE^d_UkaXmpuS1VI1oYn)kThuy4 zeKpkaO`_1MP?9j6`)5X`;MZ|0Hx;kI$c zt##dCz>cIx_BECaT0YCVVl6C|P+@A|)sGqAIp81lS!MKK;gI`7A1p!+n0L7THIL%0 zjD;O6ZX@nIJ0^Pj6Lp+apf$U}+XMcM9G|cDxM^eRSGPB=^~}q7UqUj4kGr>(E)?mI ztD4=IEal06gAD97aByf&#OgR}3AqaUO^x|z`x#%t4pWn%e8meU#OgZ2ssrqqzIA)6 zrcCrdcAmckZ_gGQ?WgAcS4IPbcyJK9;!&weM>Ew}Hx-@HBI4*hW%c={BaXpgRg2tU z(N1+k)90No7dV9F&S+dA)2>|g%hHm%o9A1ia9Uhf-y3#R*67lDG8sxyoY%|Td&Z>M zOpuP=sUgO%aA+qapbzR>iO}!j^AKp0v6dEkurpxz9qTA-<-BPDdzDe1_j!fYYp(X` z`sfovz{g1im4$w|`UdvWBM~pVQ2{O9;;uAFN{HrNccp0=&RJc;L|$aQ@!8sUxKN(Q zTd9`0CqZMZ+!dZOA$rW530YUxRZXQal@(a5VVu~Yl&}x>`T_skx&u73JL9q4A(9nQ zK5rvV_6t_K(XbFaVd&;h1VzSYIY;Q5XAM0Qs=u+08Lc@& zjsyF=l$E<`>TofB#|s$}wdeDoSncGEx3@FPrgZ3=s;g_wVv&Uwl4Ph^v)D9dC_6%- zXrVWxFcml10|u)NET+qqK_lDnF3~a*=cgLzusd#32XDpg;PM69OxU=J#BMT8^?8Gg zuWDGN-UnXETD2DG5V2b&h6WBal5w&#Q)MoF_rPE-{>fdUx2D1076O?Le7Ix1lEnk* zmdz?PpKWn{Jg<8e7wy{43)kR*rulFBqq4}?kC~>v7&vPeV=SCeaVk1tFiw>knQ(-2 z?frCn_Cjgc%<*sAU#@#IVx1{b=-KOiH0j$R7qgMEt&aCbATm^Nmq;ZQRAutRNM-mw zoNBI4fhHgQSFF=4ug~2%q@>JZiv7HuFNKpOWk7wD?qgQVZ>$%yXCM07*^fS`yEv>t;=MfR4tbRK;W2aI@E25y(ovD2f?uu)54Dli<2_b5Ku z+wF0-ZB4Rvi&lSSD~%1Zt4X_?L-^%uh{%$c(|@p*3GB0koHWfET?0zSe0#$yD~1*g zlSj+a7Bl6dEF6wP$OLvKu?i$)>L6(Ey(`)t_eJ{VLT;)>$*>&@W1ahK>g<)kuZcjJ0@9$E0QBql6BM1rhXh%HF7)(2<6v zKV;0A1x7TFE$MizcZ zt(KcBRNVc{@YdM8<1Ku^7dr*vw8b&o!Z&6^=vZ%!xA6$*!0|$J2vQOl@DP z!+L@%Sfuzno35UEc8d$Jq@!zxMgq-ZX71h>RM9(0Zzpp*pAO2~XFx5iilDKx!Nrm0 zw^j$k-iLR%Us5{aku(H<+V1jNa(tyB#eWH(D|VYvGMBc$TbjiqYCNExIlUOWkMtI) zHttn0I9ImjJ$GrX=yU?XqcT?tU0V*e2Q6?v^t|#Q+l;SMd5XMdLlvcIKRjY(TmdZ- z{HgH9Dy_p1xxZDO>k&)J-1RK$6%Ag0!(6PSYeUe^)G)OpZI6w-C`z293O2=1j9gI`W_1mjL) z^)-!%0FxO4Mgesk9x!uOAN{V@QKejzE#I`r*nDw2UiMu4P}keThgBBKV}o@1m&W3; zAMdUn8gczZXpD?H^eVG@Msx};bgd!M9ihPGca_s;zVKeeolkHf@9!b*gPQY+0P>-* z;ceyTeb2&I8&b8Fo*fy@cQBj0J*4owtX_0gd4&q(BwF#B)?@pX1?$?K4AKStl#5OW33(6X7iB`%~x2>8!>LGh?!5i#`7LC zO7dKq#>cO#A1G|P5}bTNevg$Xoujggl& z8l5``g%Pxf#nf;&nyFQh2MMV{B~jt;Xk3`l*cYFqK7G`Na^mRp%2QWQ&UDcAYacSq z=`0mr%%X$#w&w8(*6BB@N|*B6Jm#L|uEw z*Om8h@shNd5Nv5hEcX7i>k$aqtR7wKSaBKIm2GWk5+B2pn$(@YD5>YXsy%nfRTC<22t+W-aHKxp&fWKS3=5(NXcS8#3h` zF|JL!OEJgBqCdlyW^b2iKvpo6CLXO#1r^gMFijng!$i+)BKO!V`sc_u2To8T z?{>aja&7j~+#Q60c0)5hT6tDIFC14r@+Ks{k{$4$xb`s`OK87z6F=w-B-e^1pU0yd zX~-T76jGQ_U-!mos9eNGl4zHLF+#_ncnu-JlGX_t1Zw*xC~INms-5`UQ5Gdwl*Z>V zqy3HS(;)Q|^lt5DOZig#06%a3nRW$Db1t2i^AZ%wKhxIYD3SBm?Sj`Pj z_#E5MU?bpY&{61slf1`PJjIIaNPJz z&B?_W0bDSFA;wBuz-~sNV=v%*a!Y+FI_qs?=DGH3kodLG^TzDD@nTH*kw5aEzGh|onp9*_7Uz}frdF^V zXX1A3`oMbJc46OIthG0Xrzh-4;e6@DER?(*x$YaZQ_Kuvza;^$K>WdUnS7TTavwF^ zQ(F);SZMTlWziSekmv|ojRV=GR$n4^dn49UAAJKwD2VR>2M!we&>T^E+@WTzVH7#k ztCS}co_lMvSY`(YLsscWs3#K+)pX6q8onl&`kHRs3|btxoaS1#(o!vF(Z+Hwf}&sN zqCs9k!-fo-^TlHir_0I4RTR!Satj|qxJ3wbu%GAneB?cj`4wt}g5&8t5+Ds8bB;^E zd?UsdO!wbjxz^?lAYrMEGPr8WO^_&}fr^!Ex_uJVgjv)2J&yHz9GcqAis{Y5>+tP* z?)MvpDV%$eQ{=4qg$gzo)`5AJza~_XxfX(pRL)_obH@$NaC}}Xbc-UUv(ACws9{7(E-|ntck_r4l+gzcyq~Z6`{DmCH1!#TGdLUu z&R+xn>CFeiK2^MTPmvBBd;y=q?O>=yge8$6a8M@T0RPQfe*G6p1_Xr`UFNch;hk=f zFjQ`-LCo|e6@Ix2@}Fiwfjc<9|4}+X;^Y3AaGsnE*g)s}(jUAl2Lw;bE8htaDTh)4 zg%COL4KE)4g$%$nM}9!)u>wy&&~f^wTfFsv=Iv-vwVH>q9(|rvWt-uwO2a4zaI+1hWE#uy^Oc64{6qIc)|S+a!aIh|_?G zM>9)bM}b92gQAOhuLI{f0YQJy!NL~+OWYMn|3=ieWnlNJ*2_Fd-6UsP2S~q}UDHKu z0>{q$cY0I*qvQ7RBuGN!LKei87ICxGEZ^NZlZY75jgsF!W#MzWG)Qx5Mbv;Hqix2vgbq z>*s$@uAfY41U|m%oEIG+?#4|Zlj%C<%fgPXivJwZIcK zg&cS>Rl7#8A}MY?ME~p%ncUFN%l=au_g@74Kc#X1<$U{3X&g~V{x6iqfo$$A^FhD$ z0O#3Yg|RH1hyomEwMa|Nu7>a4r1E$ZgSL9qA^JO%X=Z(;NpJT2er!#RM=NL5uDh8H zKXsm~h(SMhbHXz-!MA-?M?&Z!VEVH6uHnbDS*t2wh3F*D=*}d&JQ2Vs+K;!nlg)=g2wLirmN%1=^x^eJ5r)jF zI&Au7G2N*)M|fI02$8alyAeMj!N+LK927cJ!%aM+j{OcQ7e7V zn(g|WXO9MqFX&tevQ>L1Wu&aSBLK{grH#81;Ec7?N)`k0W1~k5Gk?K{g7vod6HUH9 zbO#;yR>%5n+qF!JJc^k3I4?%T+g@LzFl)I>t(&Q?zc=s$W<{){LP1v9DG^_eJ|cA zKu4GHlMftw6n90S`E=*dYia(+0~=>p#7v{6oSGHw@+TIovA~+ztf9&P8$@xLyyg%T zz3840uyn>xMGN=D{#rqdtW&~C*A^Y7b6Cor!NdBP95cf*^z zl6UPX<#yBWCVNM9&UU@mVC;?lMV+7vItK&~?ke#^dtY{O!zLl+!?W-Ko2{C@*2p9W ziRgsOA0iTKWgN~)lW%KR?E0{^9Z- z#%|}P$|%c{W8?;`k0i8hWWsF{kJ+(bxgd`|>5$he@!}+CU0f;G!;H7|v5`!(HX$qU zI$HLaQqF}jyR2A%{v6StVI&$r!mLDi`n~iDRbl{SYH>=iYDc|UNpJIq4t>_bLHmJ- zu@ve?)~UyQ-C7>s&DD!S@ch+q=NGRp{8ui(#~KQ)nF_ge8@RUS_gn7#1$K3I zKKoZ8&)Hyt5^Nh*%VyQV+@Y?w>sU5qDzstjaWRgti64KW09xc}@jpxz6(K{NZStN- zVO7&U98buWjCRmNKNNZ^$8(szH(72#Q-V6bXK0P^WxO=0$6YcC=NqMKyiR#;nnKAKylb>vHcfAtr!| zidkK80CE|spKJnfn~$dNjdec?)s%~vFxGWmp7}UyLx$>J=%Ij$S+U>9P@2vY+&bv& zj>}ejYqid{@jm!>TLfR`@OgCY4EL+C-LfyKLY!_o_pg!r@M*w=QFa(FpTSruaNNu@t_s^^GXDx1ec5U+0)ggjAm0R!_;G^VUp1RDTJYR!p_W`0C|>4-^~$psaD1$SYuFY(-Nr{y!^gR3 z>`T?`cMd^^MK7uodh52FvYLC%jy`A8Onbh1*OO(ZByrHUU_>wcWF0ubtyjAm26bOo zkXf%h`jp3?>ez{?$8c-7T*L>^jJ-5T+*9K^3yJ&1yIE=xDiwOJ-R4x+xp$1bhEd#f zD?jU~mO3ZH-kg>#{}DtKMNzf|5Br6Hz!(4lHyut~g(t8lL`1d+YsrvVr|RQDm4yMz zSDEvVyv?xd40Y}g;-%tc9c_G6Ja^~pT^P5o^=`}|C(j}WZ_PVI0Or}uDAXeQUiL(F zq#f^slp)_L(#;`gX8izslTr2J#p0A487k8GM{Uyum@0tCH?H(yGQ}SgM#Yc!ObNM` zh-CGEO-MrBBa;Cbm`E~Xu4eM%8+Rb5gfB>DhKa*Xvt6C0483W;7{We(YBoBwxERu2 zwB%vkNA^)}n43(P^%2mS7v<61!u;gEQ4vn8&|u`=hqn~oV6H`rtxt_cK>O0Bokbpc z)<`;|_zVC$t|QhDLPA?`OPa$?4kfWn)e}3n4Ng6J8dYWuym8((mR`xPs(%d~2*@P^ zs)gOwi=F8H(*1-k42cJrU9ynTtJ+Bh6oXr1Z~8kF^8tVR0a*=bxnKNR-9Wz6DPyni z_S%?M&$BW4{0QemOzNmO&e1sr)c7Np(hSyf8@RD?JC>xiL#DgaD01|8DUx04Xf-$2 z*@D0R^`)&L(G?wg=wg~DWn96nSYK;hGN}=4u{EX_S;^h%94{1gny(Zs@?jmrk)9YS_bWd0XW3K-)WQ5yZ zzk(1)$#Wy-1zN&X_gXk&m^Qvw)->Vc4){-=mAINzMlW6f{7l4*FA;SaKL(VluuEfF@PrKzc&AzAupm|#tA9_2rkR1t6hDI91mu8UvaGcSuXhs7zWBfeaFT)nJY zr$kN`yu4{} z@};y#rqSuH?LGa0@AZe)lui3U)}XV7b?BpOYBp>^I~FNdK^Mpiz`y~iPr^Fdm-xOT zYf7qBd#*I{SEkb>$42Uw*7zmLodI0J^*5IY0gOScflNs5EEQVJldzOg75I%*aSy@* zobfTttHcE3P+g#AXu^DwA=3wn^8!pAf+XE^#2(SotP6Wh+$;lPf;#y+&~AON2yPMb zgEyWp{7ldO?PI1h0Vbf%e!CD{RX_u95~GX(K9Y!v+=wZFQK(#2iE}_g!AuaR*Uee? zehWnWq|zNCK-6E5Ma*h`3eIlMo6T0gUEVKXq>5Or5>{11gg#=xi)-vy0{#H|xDS%q zXW4#P2O!IzSYb#CaQh72#Gz(z9`sV85G4Yb&JQI?-UL`h6_CLGwI~s`90hxFw2rom zG=9;`1?atNZ7V`d8qWs%!2eQ=^b#q&LC2lmxHJuVK>K-P@ZwGTWD1hBhY)#yaz?4u z#5w{WQ0rMv6_yiB@;1~Xp!O9yUSGHYKJA}r_`jzL3JGp2HH0Mo{&>zCRK7y)Eqv);LaPM$HbSZ&%{lSK}nGNk2Br{ z-N8u7og5sf1^j@h5@^zf$OYo#kT7`k5154t)bU6OC(r@4VCoB)d)!u2gtS_y=Lba{ z!$_?wU||=5r&@mX&Ci1#h{f+#>JBV|@+;nwyg*j>0dwPfc$I*-|NbUo_d1|?#E29y z1?nW@${)ygAdqkIc;vl%U_ExAgOtL`uiXZ)vY6_-Q{ctDH9)=>2jmWu+GlX55!mS}B`=U|OtC?1 z_Ct30YEnp5q_EcJ8P1zQB>t{!K z5z@rpAT|qw_v@PS^Lrja3h9U=crhUE?-02TL>aIe07-$OYf4M~|<1bkfa7qu3}<)RzUXzU@2ihc2`l z+Io$J?QV(QHgG#r5JB$NBhf!eIcJh&2ruUq8yA1t+>~Cz8M0tzkZ}dUK+P1csx+l# z?{b9l6%~AQuzY@U9)7VYUu2;*{>vfB*bD2R$x}f@?t2xYLIB~|k;zd?s3Sz0iWYLH z4Vy=RJ}_^EN?LhwEvg3)Px1#I9mDq~$X)hw-M6)UG8FkdC1{GZi4eY+mvFFVYOI+m z5*9V6eo94R+Wn;W=-)u6^l@8R0CUn(UaDvq`7ydS`Gjh`JDV#cX<&-|b>MJ}x6a;7 z;oRp{cHJ}#a#cpGYT0F2szgVsTcBE&I=@whD_|kUkx@nd>J4no+!^YdrUm3^m-R7E ztsGt4?#!IqxaI?b;q1ZwkMA#tgHG?de{HQiz#_dqB6~tUpv{VB>xP`_Fh+N~Fz4Ic zhr2{E8ut|I!vU64Sy2+b`zDgjN=!3*59pkYeAU;)D6QnNX;rA8hAKCR=tNs`P8;j$b(X&y&fjI#N}S*eyAE?C$P7*jxe9<;YCCyKY9V}K*C zZAi9o-}XT)NuDHz+;y4(r)iPkG{iI&&l;b*Z6V`4JUJjCesX z9;C8|NwrXs(uZQ39;8#1@YwW5zY2uI9zdwuPS}fnG z4g_Vha&dCaQl=6xW*M1;d3v^v*}ZPHyq@KoB5~DsIzC?P1>r z@T;gmR)$nYd@E7K{`Ct)Wqd=~a7J2Dz_>s_J#d2VsufxF4bO%C?Qi<_w0eWEjitB3 z{RDK)40DC=qXOiEOxT@|SMIP7s1 zB;Sg^(I@yw6jtiu-r1~_s(zoN=oS*Dl}b?A!_G6*-Sxew?~o3<{J`0JAHpeKMWxG} zpo8;n-&WZdkzVPi_TROx7=8U=uRdUq_R#0d7q7x^nD)W?HumHLw06_(pp#9&jwY4Y zIg`VgIxl>uK_Ib9b|$&BTg}=I(ywW0(n|UNvg9PJ4LMK0QXW^ywpxF`%lri8NfE6_ zRg0YN?8 z-b{Zq7_TXl<=~UOlJqe_wvfl@W~w#M_ixNlvtno+f7WO>I3df_9D;zLsYTbyNHO!t7Q@E_&DoM7lu)+X!xcV3?eN(p&tfbOzgkI`G|Rc-SC(dS zPIV$(un}^Rh#*}lQ77%X21n(hSNWJN+G@0i24S{q?{T*3Lh^n0iY4;KjYj0A`T>8m zLh>Zu%hF<{k5)1LE$7GDCDjjmwoH#Iw(g6UL&s=~tVBOT;Jfe{ii%<{2&Zottlz_g9X7rCF)NJBd?R=!+@*2kc_%+Ve zi)Pb`vvQ||gnf2;{dDRxdyKSkt~tnSfscVUD6coq+Zfe5RA3KRoM}>=PNn#-(eE8* z=%SQ+S-}$%Z)g7ft)7dn-eZ*?>5b4Qz55Q=W+||%a$3DUIH4~Cx15`6KACOZurT}l z>)NT^=<*T)NZjYkQp4vCg#+eX`&IKuIu6L!`Pv8ZYhkfc8y zHliE4L#XiFscHCA1?F2Yt~;I1%!mm-5uLucR>o=GUvOj$Wctp06|-qbziHz4z)(^z zsux}7urOsbL}@v-?j}~;^vT!R7x%TI8&sx#5UROk+U@ugdX;33^fyI1k^2wDeUk)` z->HJZX5X23y@^1{aCQ5Lr8$j9j;H04d&upxT~%qYqC&c|T(WvMuH}8>JmpryP~77{ zt>7rJVRMx90l#%h7Rt53!}s(L>RD39RBLM4bF05 z6=GU)qeq2kF4XJ3VoG`g6 zj^(Sbg0Z2?4MW{1mm!`4Y4RF|uX!sS;uW3tm$Ssy@4YyAm}8`nSMoyU&|>7T(|92Q zYMJ95s5<=<1;^?RCg1nkD^hQMpJ&UWD?^B3N?u?5(ISCr%`i$N3 z7R2%l_Do1W_Z)A&`}QvgF-+4$E;e)EN{XxI%`(VzE~{S4OsRRxIoT|CSf2kBVwdG> zo|;rmp^)CTke^cjCrf7mhrVln*>W@1l?ntYCwhEX_;#i?E7kQK_Snfg0+TiMVMh@s&Uhqi1Y zAkJR;l=4iug%p> zSs3RX(AzC*s-|fC(><3%0enxIgY1H4yI61U{6NmnRq(l5`ChC+JAG(E%?Gase4 zNT6bEm)x7oFN?}uSVtz_C-}NgyqF=|em~#QGYR*2do|XN*UM0MaS{Cq8Ek*QcVq2R z%T%*`d`DGOJ0^lz{r-yE1sL+N!OIzTL8#R2M84y#%<4sTw%qhQ@!Q8@~#+d_B1_7;;kM-fxyik zrdM$}Rxz*7IY@67iN8d4OQFidA0!Hn!>=w@@{biYWZf$6T7H})eS%A3&&klI^=02& zV|4B!u99{r19$1b8_1;Sx!tpTO-NXJT2`;{@#o6}Ep?mI6wZZ)p0zcRsbBaGe=oO* z-i)s^w!!O}FhcWP;wi$Jat{nMy(qr=4xX=3dxd&NL0zY%5Du4ds5rYzYfR`?v5i0Y zMd3bNt1n#6Tk|=r= zk9J8Z)xjJDLA^q`A#Q)21Tj0W5~sP8=N?TDHvxdCki1ZB-Ei$Wp~4u=VL!0fq?nU# zWFH{+6!^w0Tx(yJ@PU=#U%3E2n{&2hFPG^&_ny9b z^2Ap;pXq+VSz6uBN7vzA$2Hi?9}vu8fqH*=V}r;x~`O{IAZDal|e?7c7Y54w{H4+QC0U z^38!N;QJfN9|Cr}hgb=z=8OQ0LKTABB6t?Wff)a-BK`=;Cjv3Q`g5oLDI^mC6p-gH zNXE$tD4?guB!S)|r9Mm@fzjiWQo;?c3^g(3B16gU6I=-Nf6!K)Ui=9PwCYB7+y&Ga zTeSdBP*Em6!TNay$uYNp^UH~tojp4bF?atvplF2JqCctS5+QD))0DPKY^kn7%y;*; z&$>A6%5c#VPaKP3dHLwF&I8e*qGYk~L zi_g9!-Sv)){3$WKr}C>IzM9Y}$aCk98sdYV<-o31>I(<#cm-IJ@qdPa9tZ1c_!|TD z0KbQ57ye|&E`m$V6eWqhy%0KGm%*>#bAAn-I7j^TCvgT(@N~@IafYWfp&%Vs!4v-jR=&H)E44Q} za+X^Lg<0O2m-;KaIeu?zBFJgKM`3gl&ohgGCj_iIY3<>kon}fv2~)!0?69pli}Jka zMa6YT2d}&;-?G7xk0G0*>5j5omPW6GQF-KCCPBrkF{L$~JH896Ax>_q;^h>*%zlB` z{Ph&7(QP=4nyxPP-O~$n%E^v(D)!iQul4?M$JWny9XfeuizKmotMe<=L#kziDgqWnr z8?G?Jnjai+tgc-CWZ@-H2$*vN=FH}ad%V{?o0_n0cKX%}M^(9kb^UXk9#@Z}rYpnA zkM-U>wU4hesI{Xv(pfv{5{5LX?bJHV#U~@iTd0`Q7l>K}a#6+JPwe~2%PkD1otjzZ zjH$kIyYlds1I0O2Y-Qr_h&oLTRNU>exx(cdlWWgX zyQAWl>6dQ_+0AFK_%9FdYgy`kU2qsb4oO-b(~F+D?g?|TMi!Y{6_O!R<)-9Yx5#Ss zuDQi2(wNs-wBFcKz}wjw)tOj2#<^m8Z71IL+_!u^%wLdT&(l_Bl7?K`s~(=p9@wCH zXs@MZ@-|M+;Hw1QQ5M@^H8Z!j5K%EkHIm&LgRhXL8J$oaIO3n#I@K*a3vxPJ$nB@f z(7Skh&)3b>QR4dn$m7b{Qy395vPP(mnatfmR`QhaDa7OG96r|3v989Q3o-&zPuy;q z7&ZolEU##RXW8ZjfU@`G#N=}nNBLgpPh>B}+Fp4UFNUSg0o| z*ZGQZF5VI^7!9czHF*e<4em^;U1jg+nA51uWOJ-zFK|2-n~Zu4ElS``YQH!>i@BBE z)o-wWY@QllU1J6{ znT)E{P;{%H-Im%|cC?YEMTm4!`1>)A!gqOx;jd79`i5)P&KW2Jm5FWZ6P@J4d$wu! zPIM}uMrmh?9$HLAY|J{?Tse)HJTaHN((rZuo}q0x#4}7R&_6H#%&o)+6B6+*I(ecK?v&g#*Xb3A*8q< zhCE!66ndd3VF_P#V+$SX#P0YN)({bf3YpEf?vxXg^L&ptCV9Thn_v?OZ6S-hE6crsPJYQS z?)o{5x*^Lb3l-h*Wb`X57S$r#ywd_LTaHheUc|umfNv3srEsi?n6G6vI22p9fmQLW zuVx?JeyZ4{sdEat!Ei3QNXO)&wUO03z7g|6j8%z8Q*XyqvAEmKOVCUy$8yAl&|Nnr z5viwO8LG#z6Z7IzbjRf(PE7m!F_6VqB*NDc#sFlT$GU?G8RIMS1|)g zP*nNyAEO)I7q2`6pwanfRf5eGMje!cPHJ+>f=PTtk=<7+cfU{2oPOOxdCxO$=`pGm z(`$#rGN=f6vo%74lW;<(C;F+w@dM25?yVuza!|m%nmS*qj+O@J`3=&({z59cxHOrC z2BpXiQwxizzOjkU*mTR;f;P~(jxH!zb}Hk^<|Rm93{6o4P4J{#^8Tx9QmSVPgPxv% zuuQz2FYOo9G1o>leA`$^SED(872BSR%~@n48*FXR!Lkq4-|QEW zA98`WocY>K)w^~RxX81QJcw$^ht#XyhSMeQfI#T23jxa~6crQsL`}@>%_0pSGP{8$(!Z^t<(-E2L4oQP` zqB0M>TDTAV_&+@-i-Dg`X`^_(#_GC6KwgbKoFmkB!%P7of-wr3^r%=;+1gMIyS@}+ zy69i(^f{8k& z+=S}%rsaeJ}rXnrk2&IPP6cQWsXkcT})#Xmq&4j=f)Z#_bC?S><0*~Ut0*f96 z4xln9jvehWT#7>XlfT~Rd&MJT!rW`;E48)9FO7IA^G)I;VKf?r6GUBSqp}m|b`~sK zld`*IHU>2Yg+A9!ayhf4V1*;Lm;AL>7}}p#uJ9&L9n7PKv=O%nxA(WiOv?>n@zgsR zQYq%FIhQ~#pVBoEC3=<6>z^lEea`e;C+4M*kZVXOjt}$(WYpU}<@ZfFP4t3{R`@6h zdJ&i7{DDN{Jdm))lhb3_lgtYRXJU2?&LSfe{gSo0lApavT)e-Q_$lrtL4NB96cA)& zu6^g~-bSSTD#&Xf0}-~rPsC)~Nb$735#6m9ieO z{-nN=w=zIv-F)ZM0Bi8CKXjWCG2M7aBJ5WR-g}h20TVc~{%^nk<6peMGl{>I*X9Ag zY6sYN)dh*bSN`;}2x5Tl>!JYSHlRTWs)C0uQv>khe>b+je$N`%w0WR`3>3$R{NG>c z<+nyOgMaq&qW}@Dq<9JGFT{OYL`tC8?LWn6KxFnyz5LpV^ew3$4Kr_n5%PxG{WbJ| zGbhKN0WW3tqx+&0QO)|(uXSG(1^kC$tu}WsA8|y0|L`vn?mmDCx$`ICt`6|omL05V z|MiOa`_K{QU`25#4LXCl7lwi{9{V-I-9xbae~^BEV@<@Kd;g=brs7D*cnbVL@#hI9 zU^7S&T~au^z=A$U(BOw51xD0yF@WO0VpF%-9zz9SDfX37HA;r&O z9ZEYQ%~vD|MkES*gX#VBOGM#JB2jqc2T}MNv4?`n*I!|sMuAO`{A?6KBqKL*)N>9&!3_g2>I-Kys;Z=3LUGB873Sw+5nDfBOi;{ob<%HVZm28kL7EPk6!NIm49xaug_J37|CG^)`NI}XCiHiDDq2QQ_C?!d`KEk`d5H#f|E8q{_= zTDc#NirVpYKJvWQ{YeD8WWuj3hKM#Ar26~Yj^<%kre&Wjn0X18%TQ-mxT11ZO3nZ%LxNEB zvQf)|)8!{{v`Owmjr9QC#Vv`L0>FHzlGzU2+DE+AKSsH@y^UNc9ppsF(%cVJA)>6- zriZnNC~Fwv4O}{#T=%H46fPwIeee0&RmpQV;9>sJ4GP@KR}p>ej!fUf>d>w-m!$+) z84GpeBv0Ui(^L;SoX8;HQMXGB|;5*VEad?KDxor-Z zn8Zp6@jYKKF*AP_S-e4;K?B0$OP-1;_H{gAYA`lT=s8vQ#E)UQzM54ig?WQkYYZ6< z^B!`@t{bfmi!e@jn}_SSFI(-knpax#5g|1*H;oQq+de3|*mT3eg|!RPf5Gs5sWNTj zMuJk=A3TJocA(iQqVqZ-Cgi2%(r|mPjydCd|ApkB)yS=Z{#yUB_EFK6Eh&}wgc-F< zw^K5C*edw`jbF#!$XF}54Je;dPYT8H} z=RfaqL_Pq0hU`TbL+L^cJa~0V9!6R@qN&|_LKIi8H_#FH?o4gm?ZdU|> zgV0=H&Z;!l>5R^eCWmwOi^2pJSSr0diw={wICf{enZoN503D~r;);?kF5ei2W%ua_ z44sy9!X^0UnfFpaHAq4`c>>Z0qGM>4Uk}72n}n>Zyz|5{rgc%#Jt2z;JirbR)_MG8 z1to_)7YA%VI1YpMbI&^@H+g-%WASnVggJxt!NUPlHY0amR$klrgN>41ej@r_>7Nj5 z!KM7>z&~iqH~-=S%!i+V5Z2RL0|@t7Wt$_oinq2Um|`B_>&Gf&Z?R8NZh8R7nD9XQD^zn%f2 zCUkLe#90)~+)U^@2kE~WB_p*ldN008#IfDJ*4GT%u`(zI599f&(HbFDgcKQe85^R2 ze4?dE2t1TSgA2vq%|41~pecuiRZ{OTQ{=Zp2T#xq`;aU>N3Q`1I)xGgQDNBaE1LEf z_*d7S&fYR-u})cn`CqxWSfO=LLiKGaHfJdo`>lEC+jR81d%P>eyi`%C92ENUghdkbwYAwV3WUg7<= zwx5u*HzN<&_f~&_s=jPoS7Xaz19h*CpV5B&S!r+~ng3+etq*&NXa{gstlQlc1*c73 z9^rh5S^1*Fqr@X{vWDmU@=pNO$;LunY*1y%Y7et#!dpVc(HE!7lz|z)9`uTMN zAVS`2(v81MdB&1gaYT(ao&NT9GgWAkV>0f@VAT3E_bBQ+7H`iH=)@~!#{A<4{i7ge zTCERjQhX+4Me}y=>D=R9FyDSW#|c9fIr_!(Y$hb{)xVC7tKen2oair+4Z6bkFua%b zZ8o~{9V$Ulc|3)Te_9xP_BF?%GXWEWRhr+gsER2&4JBo=t{WZr);y7a8}#+E*K0oI zbPqAPJ1&3==+BBTy*eki=Z?)#TMxyyi7lb!=C*QZ6xihT5=AR3uLDOD;lPK}V z3~d7SSC8P$hI_`BAepiknvOyW5A7Xr_~*3`0&P|~iY+E+I&*e#h#}v-xl;A2!p3dJ zTlHSZIWw)U^g-WJXikPEIpplnx4=R@;v-94%kxM&Jv3HEcd(}Exr1A9+S*83J?L$s z8;l`kEl+PeVwlRmAA7CHpS`oQX&#|v6#sdqcS*CX@7n;jSUoxdf=HvXyPT&}qVc}g z2S2o$?J17J)Hqp-w#f#_oTxs3qbY4o<0jzf*IZO%a&NNYjDhsAQYjmlT{-Qn-VB5+ z?7pI=X+SDdPx~4`MnBk`$Z;~A->`V_Oi{U3Hg{MNKo6>IE8}1XfelC8K}vhPv5;kN z>*8g>>gxg|i4}1FsDnwbop0qA$7AbzipGW=rAa|zB8?{?WYZ(}G6S6+ZaXQ3^aY&2 zAgo3Xt&y=EZ>VS1>%8rT@4&5os!EBq%z4XKO?D4qnere-C7camPW^quXr^H2W0I^0 zK8_`qo$Lcp(80L*~Z#0y~hmkW2(XU-=)_TMsblQ<(Y9spgM zGRgDF-|QkE^Np_#@zIXg7S4Tp@k#Z;M(=pxg-}vaq;VIq9`P%?%qEC#)R>MdLyEmg zYmd)*pJ;r=6Aw4Hy1gEuk>rE1>v}1r|*8+3z=q3eCBI}N9^OICe2YqqVQ;A0z zh@2lIA-UmX}A@ z$*<`H$CRrWA>(SIUq*IoO?SZP)cru&{>Px@ zY};721!8KfY25?9JI7Zn6|;AKMwskr?Co#qcgsUG1juXmEDY5|dLuPcSg>0tDT4+{ z?RwAh>d5|a6+e3gE^al#NTI{9Uy8|?{p5Am$wBTnQdnZ~c)t#pmw!%VSAmIjCvDuG z|NMqMX2}Pj^KT5<;m003IlD1%%xq-emzZaB zv5&XT@YPX2-@Lx>%xaR76tbC8q~rdeQ{14z{UUEq5yBpvC;+KNmk7$qoELrxN#zA( zi=RS)w*YjN8EyxNrHBFG2Rna>2OAdvU^22ld9W}L!22l<{u~*V8D9lfGDOUs41_Ar z?ir@!XSEf0Nt`?uoFwy_QB$s z`lagNuS5>hpPgOL0M`|DQ}icG5LhY7!^9OUtn$5{Tbx+WJ@;$%+`pNqf1BLjOw_+k z#(%NSznQ3i_gBA}s6?~#mv{WlL$25^ z6^aRbUdY4-L+tPb72Pe${R7ecRV>EIcqD&I3@M2y|53IRh>3r-g>rM$PA@+6)yoyFw zHs*I7-=0X~3@2|djmr6!Pu6#BVb(*4ioNTILCT3YEq5(@5Ai9PH9~vFyr*hUZF(t3 z^&d>3IvaQM=41J$99z!t4)gF~RXVb!2R1nH1^cLqxg3P3eD14RSwa>^V zY_}O_D;e~3ISJ>Fta>_hj076R@}BfcbZW=^wusMN$FeR=338`OLRxxMa0MSuu8dU5 z8E+@off4$e&^a!Ok`DcA!>N?RH(?OBh^F>>ZLJjaeKHL!Q%;v*tFTbh-onep;Z5#g z!%KaZ%J(i_Q<6Ibl6B&%KjihJ@_mUg{n!&3{E?{W#-YISKHLRZHgOTdB&g9WWeQ_Y<52phMKIN6VXONlIHO1!uzSBSB0O%<~){Aw@I6xl~pnvgeN(Yb~Bn zzbwmWoh&d;^q_0UD@4N!F=U!-Qu(M96K{^-J@%6q&jefQ& z$dB>Dc5R!$LELOs`-5ou0T4~EEiU7|{0^5cS!6v-EH;;WKKCf^1Y(Z*H;$MCT!H@zrb!_kRE0 z!?X*bUYk-C*FY7``1wut1Yjme8Hh}+wLV3TVw&avIN+I{=+85K@|C;hBI9uscdX50 zPeL!;v@_XRzw=|Slr0wH*6J|a?jilGkd{ooj4R(O0j4dH!ocN3jhL1ha@#`rhK?0x zTz#rf)(-J}TKTZOiRQnMOUEY+?|Um)rySwF;&kO2rO3&b-w~^WI}3Ib&fP^9NXJd3 zTn~5CQY+t;7gEFHfh0v)>}c-Y7`IDhnU4r{{!$NhK`;67_fY%0)AL@l+dIu8_Y}w4 zlt1^L%Idspa!B$asRSEhifVriGvtI@YFgD+ii5rB6fw0EB{eJAdmlNW53I-h8xnlW z-H5EoTY#dk$|Vx!G`~@OhSiR+=C(62qdKu?e5KK_aQ4Mgf!iS&b9$WZL!VR6K^XX> zW4IbMh{pRra%_g+)o&HLWd`_uZD_ve*g^BjWCBshGuej>*FC(L5y9?f;%8euL1#bo zaM5J*;|d_9r^(<-t@DJP)yvxsr&kQfsp`3g@D@7)f|kll%sr-7!~JE8OS{Zv(3Lu_ zEiq%n7K=c8=EV_aPKA!|=?Vnl{+G`pNJZiUL5JtbWQDJ6X@rg{+?wZ`ave1FV~1ZZ*l%evA%;+WN2ADf}8K ztrfQ%QHcZJ9IE+ZASTbI{c|5f!W9pvJDzJM(wVbH8|Ojuas&soi5eo0S(Y!GWumQW zn_Fky%%??)WWFnE*sSEp;n%_*=%ZHeZX%lZD0*t-cp#s>7b7g9yj@8X=!uK#9U2+1 za{grV1y$Z|CZwZ}#KC=Xw{d66H;*(;!_!aRzH;IFhQsrcpq~^QCC`}bJnyhU#zpJB z_rAw&59xg{iV>pLK7`xw_t@kzN|{lzj>)*waj|4!hICvV1llcU)$-AQ#O{A5kpiwd z;E~>IL17-cZ?w#)hhq=LZ%jQd3>e?}_A(BF2u8nJi0^b!*=wF$9A2MTNLby;qZ2); zxyTmMV+Ue()l^TUf!Bvz<^vT!4s7+x7G}i*S!|(v&v)c@r~>FvApX=+;}$ryuql`w zCqZkk7J1TC+8#oGJ7Q_Sw#W#hn2W0Tyr7g#e=p&8FSw;EWp6SSmJz`8>K;! z1wV#KD8_PRRW7Hd{H6Eusbb!JY$c{Lc0hynwFlleHg{JDoE^Y)Hz?imL%O^Df#KTE zN;no5Aj9~DCY(u119th#pZ+BBY;7A%F@9rt-&A@o^g?L7t4gmVkw-+;&He(IB3Y*I zq|tHd8nV?>GTY+i(#R1(NHz6r#iF!ZSwqiylh>{ZkvS?aHel?CGk|(#!{yuLxCD#0 zhW`MF=uFHvZoBJd0?-k})^xrWcC4OO+7IWYpYeJ8quzXBVk?97R{Yk$=Lp?X3dCf| z@^WmBf8t)fWwNpD7g};#(sfHltjIP`x69Yu%CW7_EBL~cmAu?7*#LAsxyIX9^D_n4 zaXSanDclt`i8J*{y~!YlDM|zB>l6ufZX#@>@-h$$4w!iKNhPQiYYlCj1B+%^Jau^_ zEPpv|Q1_S~JhEcoV6eq$%vYZiYkc_$n1U^(Ua4R(1s7hj6AR={A0xt`CujL%4jGh@ zIZjjnG){dLYBFd#|8@u+ zMg^&tG*G`Q%g(e6A6$65^bCFShxeM_C_Nr6wp5lu%}jUZgT5p=ggjE?veD;#B6+F<*0iyZY|w>_kbi zjuQ6I@n|yg$&#WSM`>`{dC~9hTLt&&=5&SOo|$qMm?ZO=WE%{3X|0GH)pM*%E=S96msTK8-f+px`)nr;9M>GL+7%$K zeg1j1hrZCR&xyi~i|@8YjZ?6(D|uRt7-Wc@I*u1)Q$(g=ZBvwAy3} zYjUgjteQI$K3%iCKJqY-0yY|wg2ek8xoH_d6Mn#(rhS!#o4bQEEEDrzosZA*pfg1( zN6wSk;AgLU9;Bo+Ho@+w)3u&aJ2p#<4n6!#8n$+ZHi#@4H6wihn9zP_eg<~}m~*ef zjO9c9FwcO{GYfY)i1XT8O$wPhrx4@0$enxbCn=`T!>)jpp6V^V?o|Sl*lq z!P&1iZKGor7|iKqSB|%dk%AE5#N|>xBF|nEd{)V*tx_W2*GO5(V%!21#m~uWIm>6n zX07D7oUUZENcOWmjN%(Q3LRLP3@O4&R+}DX`#}NC{7wON7cVdBlrvjP-2@a+=$Qgq zy|0hbk^_HC!da=V&I(6lyU97qu-LQ7G*~GVC5x&_aBS}hh-OM;LHaous6eA}mHPI~F-Dg1 zQIkZ}&|5R|(y(MtI-$6efUyIw{+VQ7a-cJX_&lU6O{-=PE#xXH>!ue}(} zy^iWMtp=3LdQ~m1N?Kz4btIB{;%!tIor#|Td9^HM0gUuMZJ9rEiX3rJFbygZzA>J& zLV7(sCS=m^%;d9eupLlp*t0IVeQRt(mklznua&?6%6t&b6`s|UFjH+yh_bO#MMJkv z4KuYM?2b;C+-Wm)Al|$s)a{$YCDNh>yMb}}w2!pkeLXQF*agd-qH8&`HTuT=mRUGa zD+sVONT3-+f|Zw^!)CAn$1+XxBFP%*(AOX!QQETX%Qr zr%5~)NbN4wIrLu@JuPFVi;(vBVs$|I9Y?msj^%fK9=k?rRj10ePTi2o?7e>=p2ymoO2K@^j%QL1E4-n=M&;L~A@&&B{Ftnz@LI5hhB~TP9pJ8HUxm7n zTCEl}UmPbi<44Lbafy4VZgN)0oVG?;#^;?f<9uxaQ84xjl8%tUab=vKi%4HNw?Os- zrF^?Jv@(y$!Q#A1>h&nO;aBD>ZK>SD70i?nug4k)>nhcR%aPl+NYEyp-8h?mifrrb^W)Ey8a~va z9`!RINGM%9C(Y&CNy@w5KkbIAx!h;c8dm?oxDz{J7nxdW8`-=F-);;6O#kWcBusVp zECf?+wL($nY=TkC!x2Y9t?iff=79P!lWUaNnW_o(HDMbw-lqU}VgynFN}`?vj_Dh* zKFQ~|j=P;wV|rg0n;gW+g0T#qy_!7+7_sg5M#7rBR>kY}gnH&UiEFsqyD0jkx%DB) zrJ8xAqXUV^#Q0f&z0tA~vOix}<<{Dl+Ey2aevK-EKXDX&X6!VxGlImtYVWy~6bbuL zgAJ?FEHb}5^}frak*M4fY`23wj0c!aVcZ#EY2Lka%YMF6iE453o2ePLqCC% zQn_mg)nv{l-+0Y}8*EkAG(WsVg6&OlN_EbABQNvY^6sB&Nouanz2D`IjO%Ovg|odK1=_4 zTQQZp`{WlG+$Xk7!(PghEKa6Q#syu1SF!RZi#Y#R?RlP|doq5dp;@-sKu_t(e4Mda zZL7_KLUc&q_Q47e&l#g-PW$wbBZsKv+8x87KLhD!cCWj_r-9JDNHN{LJ)sY<)>&@V~^jkEm*)US4f-F zpI%USD!~g}p*u?H%IakR)I3igP|2mo>|mR z5}xtT6lp=ktda}k6V8}8ZHtHoGf%hWKk1U>e1I-F({p>Z?QlTT0`HW7x{GM&AU!%~ z?7${t2pUk}H&XFd9QRa%;dmLESqb9kdSH|ChgG{q+noTJ8>!ClSJg3XttwIlhJ97e z1sUW3SuBvj(rB|ratmxu##Y;IdxCz8D_wE#>~QH%Ipx4xJ=cDe*igoq@&`L|534T@ zk4~_7i&}AL!nZXYtP9qhyT4s}ADP`w8AlCiNhzv%yL_%|y>aWJK;}9DoPG8eFB%2k z=DM)CD_$F-0TefNfz--m*htJw*(6s^*S5iNiPKI72y4YeiXJX^{L&6ZBRtE#V2z`} zQ_)8U@C+jhGdE4p?WjaZxX|!;?>ryqMQ>K&P zIJHEaidmQ+kCO~FmGf6lH za?L-zTN3bl$REdn|C`c*fC0B}pMB_O!wW$FLp2YOT7B{U=NCZ`Z-Mnb`e*-*EaF@L zwf-A_9;EGv`M48A5^L%o+CSIljC%&A{{AZsIq+VJfgJ4ka`4u#ys;SY)si>tA>fUj z210ji*H54Sd9x20F7P-%#6VLSf3eZl6j6y!^+%(vqr^&Hr+XBCkN*E!djCj^pIHIzo?~jft&%p?l=?DJlcmLJ;7d{SF`rFGvkBAwKZ(s*x zIg*0Pk)K{^#|Ns7b6zfisTclRMD&yEqQHbCDr@EfO?fhRFrULC-5mdFQ~lEbTIs<` ziYcE0V>BZJV?1qrM(!6XqQ8Kz{^)V@5`4N!)ByX*&CDdh7pETZ^MGb$=(prs#~}L~ zUDC8BcYfuyDPJEi>yjGEISCDrSlhiwG(J4U!twH(pM(E5Z~AX<ZS|8493)0X_Ub$_!rGXG@jT7a!PUGMi$df4BCfdlU5^kRy0eCj_6)mC>46$p6y zorxms#M|$pm;i_Z>3ydPSKLlfnG;7LAY5qP9A@g=reeEpwzlmBs=!0M|4S8kQWv#C zb{x>uy1|88l7Q>u?iF+qksw{M$UxH~!Ni$^=aS^fbG9K^S9qW0-P zlxb|6STC!b4f$X@GL3(*yTG~mfzA)LU?9GL+6KyF(%an(Hc)MDnQ=Af=7#RZ{>$3f zA~@j`XO@K4gQZvq;uxhs;Y8V8+`OIw-iZE&B=d`_EKHl7l|Bs1)dcqI3Tii@I2OHO z{%eBCGUd;L?5};Tl~gc>mwkO0quE`m$inT}A5qC`Do&_~Kt`o9n8jG;BzXir7;(C;{|$)by|d zmm0(OXZ{QOJ2WGES&eX^-hbV22Pb7B^YINzm=!pm$!Zw9j#k4`f=9-zA1e2}h-208*~nPF$zV=o z(3?&jEpK{7z>+L_gc-xT_UiF%(dbiW&qn%_eiIaDz1b*bMK1D3lM z{E)TwGa?`A6K*Ye(A^5bP3b%6P}tJoU%BgHt}_D z(JNl)jO`eXu8no{mA>H9nK@m`P68kuJ)=uOM6t~2!E++Ze_Um;SgE%lY1HMG^n7@% zF;8`u`g*#B1^`(r>8wX}d#%VuyxT=26!06O!$)9ES)iUeuY0XIYKH^Vd^4-e@&hs( zIQTs6h$SAKR1nw6AgqqKnTc|>`ZZ?hV*8^py)dJVuudY&F+9Nr^X?$5_}-d=S;6!7jB_Y8#Qo@0}uE?$G3Zc?3E6!rIb+w%y-D+>9a9Y?DQGppSE~3JoS)LBB z>ZpR{TtO&vNo4&gkZDgf{-uqAH8D);FKBVtOWu@t+2_LnYYbtF8F}z5lYBmtF9#PM z!&}VhIGct{mmi0W`Ev|*nipIm0z-}jt&KpU?$Wcq2qG{fMVvOn{&|)#bx5%YovvS0 z#+Ck<`q07{?h3;54u~H6EKqUH>(J?T|77fE3zIS__w#%3$4$;5a0qAWw8c;{Vxour zbxog^__YMBZ94Pt3Li#T52UnQye@a1zbq1nYKExuXK)oqqhDDZv{{rqO^Uhn$)MJf ze?}jk<5ww-6)jt4B+d5yUeJ63vWN_ZjkeHoe}O5CxC_3Og7C;4&y(4%aBx<|&0=w3 z@eh~@uEq?LiGH0`VK=zgB1ooa>;q+iA@e1ArQf{hs+W4O~_>nde)>$eVU={(6|ucsbBb}c*_WMo6(IXG`p|GwmDGjmjq_7ff0UB75mYr3KvYKs@$@MZm-aqnUT7i`ID0i|@4*sT~C z#$ce3izFWcDi(V;9@3Ja3m+7*iuH7axYUtDnjY3J5>;k_6&#~C&ViVTTCYaZCHGni zD;s~o1p<#H4DF3qGK?@oFAFbjT~$WN!6O^jqhEO65^c9+b;!|B%A$rGty;=7D*dj> z>`gUZYk7I<)&W#QIBM{v#7O8uWo=&vGypT5t&2eJy!F_XnJtRtK1ztjO#7iE;aRO@ zcZ}RZ&dQh#LPLFGKV$q0W>62pa@Y}GR5#Nmn)6%Nj>8Qh!amH;O&9VD!}y zE$YwCcIKgSmh8xx%`=)8=feQN88h2qC3IAGa|07>UN~?;%GbF5`G}Of`XZknaotrW zX*bjUXW$Fkv=POF^&g4>Cgr_Tscp}^Mb?2l^#??C0W_ee!y8MF);C^%`;G!86|}Hw zj=!#`yM3D+vO$C2V|_frvOG1{ar?Fz8fssbpv`$QA75{t>WJ>uz6>>n7LeYUu0TK3 z$=$Kn0IL4EMKWbw_kbn>S8ylh3kz5X3EmG=Ko?XtBI z$T9|M9K{hb51Z3ksIN=pvz>x?K4Nyi1qBD3ZKcO}ouf*JAefBDfOec#b*`x#IM1x! zWE-M%OcEJzpZSOO))53ZB}j;G?i{A9&5*z1Z5cahKn~uK2~vL;JN>Y>C&+U_w#rxW zjmNdvFN4C6euAP;4Kuqxh5lw>c zW--cFsqk=5vtx?Z*uE-|NamFKY{fd?PtYMjFF0*0f&`o~YzgCipP<2($FVgupVrbs zg5G~?n&3Pt!NE8o1BZcODso*d`rJv{>vG+{x@E?PFL;FR_#>I}Gk5kAG~p6RtKEx$CO2~r2-DuH`8>F2+l-OdztEnVsk^LX zhO-^1xU_F!PFlnf+}8LutkJ=(sJ_UCST()DaBH|Ozh$#;7;_dSEv+HKE9^r=NOQ5X z-1)o;?<+*F0W;l+^dX6>l98OUmTdB#iL;%-lmYD#+dMjfmD%rdOVbkO^c%|sHmJYV zWAkzm(NeN=SlzprgyqH8JNhZp@U!9B5C0x4|7V2rH{3=B=r(3^{&hC>eB4>A4H14t zkE3jM{jTHvFE9EFruef_?`a;QEM*T^wUII-Vvso4RIlEns>*s@r5db2r0e68SBWRi ze~+m+odE#T>FW}Q!8cooogRX9G~|DF5xA>{3`9+pI7kHXlhwZ+?hn-InF0v-CG9ib zfDwsT12F7uDEm+FDB{?Q#lQd?)CY;6Z3|H26h9EvBK_$A&7=TQwDD(KdoV&_#rr?M z2uOT_xup2SNKMQOd;-v3GV;jFUm0T#c$`d)N&rox4}+hu>VBVl>!;T#%YzBIsFHIE zJZ8T!`0OF;^9%FHrUY^qdLJW*R_Yi4VGNQ2u`>NjgcW@_?tg7=<*TRc&sM z+)y9n&kodPpwzm)6v%M{-6P8(bj+*xoFLg|+L7b3HKG^jXSeBO-yWw%($e3`z8U*S zxZH0xlBR(%Buu@U)?bZ0&8eA(Nu66QF(p)~x9@d#t)K&}tgcMrl0@Ci;^N}+{;oKq z%}RTsi`0zuWjN9$g4ksGhy20{=`ZxprhEgW$ufU*bNK;spSArt$o&H$am@$@H#74A z`(o(``!95JDgI%x_=$-6HGrruUO$`j^P9<-0TX3?j%cFLfCJra_6uwC8`$H;(ta74 z#3fw=HbrMB)8CBVU-s*tw}v%9)Qd%KQvh)v83OqrB}B?^ube<<@C%m$@$VeXtV%vP zh!ek*1X0UV4{8q@q4T^JAzF`r{|ks^@D|*w96vt;Bl95w-V!xX5WxKVt4N^X;4sPh zX@Hml`H@pfvH5vU=?8+mRS-keQZNTEFZ{#e&u#=BK|GFVDrcFL!FFE;F_wFg>Stot zo1Z*g@on(R`iW2Sy&d`6U%1mO0Cmmad$)-_-HTno0FxN_|M^Y*i>c{;1unCFd6bwA zIRG|u3q@P;^UHo9$a@A#;AV6{tLE7+r9--j?Tk8IgMRNe;sW?TvfIcYxUrC5*l9o9 zw_ia9Sa?sswvqoO$>`Z~N_3%-hc6HRoe9KfM&rABG}{AQqBjSg~|k!WMgKnP_m z{0mRD2g)e=Q;-^(;N>kErGh4JXD3GZSsFiJBAfDNJnqn+AHNF%mxq+C#@U!O!I!E5 zXn81S1FH#PUcFj(c&^RabKk)KUPAIh#At$p(^?DO377GSmFccI=oRegc{Z`Zw@ho~ zV;l_cFTy4b{`Top#j5$PvYZP~51ZJ-RW)VI?9ho9qcf*2R^n^N!mu2sD30Z^TE=z$ zu(M-+*Xo%@T??GYCxr7Sv$oqdP$Np%8t%t!6|3tN4i<#AIQE$6K02;$RfKleS@Z4n z>5T**y#E%`fT7&j|a2A+^T;bEHOR8YrieJ6*~?}Mp4mw!Gf!+ z5>`!tMfsGUR5{qLy$Y$|rG16;C}A`@Q5n%!aZ1?hy(i45T^1hY9PQV{tvvX&DDW~v zzpGJ8W#oQxlI;re`L%Z1XbK!{Qxd=Q(Xn%6&;*h9CIG6Pudti8__)~IRkIu7+G{&9 zf53fzYWv|ie*sxyopT7I9+_idj1+am2u9bmunsGIAI6t}<|gOGibk?|24qk2w$G(N0RONu@jv*u#{W_?uSp!A}np zRm{fO^S!{s_TNmsM1szUFIk6a>iEeS?oJk5mXL98@)b+0aTa1T#>{Y@^HkBm7^gR9~yVSfFA^~=V=a@$}1Dpv(SMz zuk;49t|3%fE$ncYDlnT`f&NODApW4E6n=?r^fJ!W$9Un^z{v7_NsbPyvIy8&ZT37V z-wV${Ba2*g)C!XIj!7+JXUI$3xA!hh`F~huF5Bqlu^^JDZt9A@@1$8tJYVFiMu~6O ziPe$s*Pjs=MsQP|>43z+xRWIx*RgggBk${9z{x!HOf7|m63)qi)S?aB=8n89v_LLj2lrSB%|NA@d7@*C4U7x|;}+wVn<$)eWZ2JId!NXN{n zrE3R)n1(9fr^gr$Gxfpi@0xku40v_htQ@*qGk!qhah8Eh`9R;GCwphusq)K8!Q94k zMaU}H3-fpdrMw9;N-grIc?Hp(mx5*rqPAabaxxW?8k^Kg`bh8H7T=wp7B&rA^4ybh z-HWuMKXeY^N3=Jx`X|5^oDG$!f(8VIz5S@-v^%Zq^1wj0+{Nz9&vsZcLb6LQ)(Ozj-04WU>0WbeUR3!rx-ki?ze$0wdS31sDd`;Yx*r9J=0w)hjm#GnQS%YMGhEHVVJtya^u@IO3&HPVxLaEwYK}$3%0ZHPyD=e(V2 z0_Xn2YU6#VXLXY@1~|uE4Oe%6MZX8PU!WmmHi7#awurfWt!<>vQ4(VJBU=tc-m1*CUElUf|f*5%2vC$ zmuNZWJk-t1OY6m7I&zZPayP+Ii;!qPeaWcn%$6hvqu6(%>1JgmAewTW>W+t)t=;Wk zY?j&amLm7xyisa-CG{g|l_)eI%;E|2=6hjGtH~>kx^*AQvRY)7Ugx=#b;A6ao7PW& zB*jt0o<$Ecch>xoT8plh3qtaPZ5h*Cd~ioy)7OH6x)(IWI!2XK_b$pkzkf;Ea7Eti zlko~7aV$zdPL+BqrNd{5&#-w_G^4?`}PEOw$~M_op#l$ zr3_8PXge%4zpA2c>I<++j^yo--xh%A;r*I}mIa=~!IUHgt#WhYj}AoIiUqkmXtA3}(d@hMHdBorRGd$4`*Cg1Kvn7(yJcqzNNBy?AS zr=`bpubx)WSWIZ)QErJd8}3?rXHEuBD>LW$=#}(6NOr^sQ??PNQ+k%)*GRX(_ijRl z&%Q{Rnk!bO161r(qH|YG7EO@*KAWRf{gws_0q7g4o<}%tczs0BWnw}&gM7-^W4f6B zz^j8eoC`%j#iS_4jOprhDQ0Nv+3*_F=%*W8m5xqU{2W^)i7Npnm>(eu4*El%T?L5Y zj8Q5-2|D>wuGCMXTNyJL*j+w%b%;#0Jz$dhBmFg$z;FEI5H3T6G}wq=`o9Zl{0_4I zyMiom$ewve{V!mPe+~@G!P4kInL!HnX(|CMF8qqD;Lk7nAsUD5(*SGlCRobpzoc~O zDi83kOU7xZz=9R82hryQPUcryN9TZX<@|t|fAskM@apF<^I>o^Ct>3pZepNe0zUi5 zD*XJxC>H_D>}2|}4TwO<&=O;`ZpB}~wCUmiM0JvyxDkcHA^!65pY2HC9r45it*U9x zd9at=K`34Aa*zBc`xsnBW&q-EyYP<)aEER<8!9)RuU&*I+u;PqF^| zBC@N*z&*ZDjo6f#%_KD(=$mJ$(vH#n3ex)xNRi}d8un`Ze+Hp6_qoCH7Z(6b!@qn* zztPIS&&zML@}HK8*?&`R;I||F?{cL76|MZ}O`?j#c~`GCgiRb2C{4I7C={`+rh7Ck zm^QEoA>k56-i3;H0pM2V(G>p@UnP{Mh+mXVZG}cjUs9MbDWTGhXqv*VGYow*^7uCD z8W<+F95Mu3+RyJXSJPgpKsQNDEKVn6@V;#Eo!icv*jdX<3yFm^EWf_4g}fXg6ufXA zD;8*7E$?dZ{eL*w)p%_*=olKWqp3$MDg%)4kY6{$>5B z?UjVQJ)jNegy?VZUEIDwI-hdPa;5^qjgnXw7$3h@2;9&aI-h0TgKIC7PgrX6rSLyR zsdfrtZ>zFd*`JeCb^c%NeP>itTi0%gC{@6Uf;0<=AVHBLEm15-snUC}Kt!tanus8% zsHk)z1mz$|?;tG*mLo_Jq}QOKBfX~Gweh^?d(Rl}0q-5(z2lB=_-~J7?>*OEWv;oN z`OG;p3zD$z#_-rQ@9tZZz2O5YbH)*-p~{F^_m}$ka)lJ z?ak*TZ)3Y`Tv$=(ZyyvhPz!2zK9QQm z$rC~pf6H*{AusU=wExO?6O^yY@Ndke2OoQ6x}E&ixOu#BYp7kkv2zN&>yJX+==VXT zJRJcja_kM`IY~)w_AJJe~h7LP5D@8DVgPF-J77nO5U1Q36UPZXN|BJ z-QqIRr`poX>si!dMqfxVoKk?=)n73~vNDAf&xX2TXDet}NXl+Fio{q4p zuIL0=nHyS_S$O1fx3)m!X)7eWv%k#g&h$B&eWCi*{s;M#(WrpPJU_0mNrE`hVJq3=dmm3O(J!(p2@auXY>jgP9V4DXkp$qU=m8a$TwJv1tT#P+8sFj zP$ytI-{g2p?|VQ~ZN(?BXgBNzQ&Fs#9HuYYxEJY_oMT#5LMu$D5@x>Ae$l;AC2y`q z3G*`FtNo7GeWKB0(RR%+4w+kJhJ0_On_1z<0#-dEWhJoI!$HN`)vujpluhL~4O}bZ z=`)|18Cb=XBwTpAS~tbq1sJaO>dk8ZNoWkafb-2b~FxD=&j~&DJn$_N+U3yfiu6_o{m+P4E>(*E_PvhzP5V=86d!Osf+& z*c5gLhjT1?W^3C&n89(+-QqQ46|pM?efUnZ3wFgw-P`M?YOl9tFwX`1X^KAv-V#Dv z%45_=En8UDCK;Dx(>>dEarH0-*blubBYpPmB-|ZEj#tV7t{zKihNnR)Ve6@!Qcr=C zIqXgl)JS+`k(I(XZx+Wit?OyJ9;_~})pu%sjq2wTjz8qId3Ig z;B=j^Z9N4J1l+aRfP2j9rk=QisfzFE?42uzV8#@8vtuPu$JP7eu*Q_cqb?H$=NcYlDU{FNfXT>|n)4jla+pYgFW~y|n#mLu`3XM9KM`6n zM0B(sB5bbOzAHa0NvjR8VkfwoN-J11N}@DZ%ZLhfj5s32sFG8=2Z!Kd2cVDwJlA)n zV=DITm6h_b{EV*gmd^WovolN`ofJw;Pg@1SC9)!4Oev-5rz7`S(nxr_|Wpf#Z| zn@d6Yc*!!e*f$8=Ygt08!v$H~g&d2!jx-Im@h|~H|JQi!b|n5L;uA(G13Ot}(VL7wMd-c+T@oxAQf z446^NkmAA$5-#<$$twl{yM%bp*f~HHql2t(4Wm=3mgf6Qd>nVeG^UH#0s`108d&%@ z!!T1iH$dNPlmUg z)XRp!2pw+4R(%mTAv>CnnlP1{z1d!SNZRfZKVGHJgxI+0LNA-!C1Do?(p&1JnZ*H$ zlIQWdhItKrP=j2OO}P`mX%-W1?)x80<;>PlYDMn1pXhx3qV#rST(X~2mZ|KucPll8 zx;{&tPf=)Q(Z-8TWEJ33V{$jDvMv2frnyIkV{ZQewf%X_)7RR{ z0X~lOdZujj$#G}(%mgt~=nB_WHY~+%43BfcHzYEtpHs=Upm| z1Jhf`^EVy|`R4Ye3V7^C4dlRchq2*dixQ2n^!CwwtIH*qu$H>^9cdUbP~T+|o4lhV z3^j3}TH4#mCqL_W;q$@P4`r0o9)j$oUd;rOk7zF8a$>|KV0b*r>z0iJhQ|Uto(;BJ z%{82V5HFHp3|C+rCy&?*nxzYmACF6moXsnzX!X5}A zO2wz#ebuiLkO$Os>68k z`_S+bx>sk9M0W7U9*K{TskW!bswGhi;@6}p=|l0LArNLSO}HaP$Wr8}tMJ<{wbOD@ z8*w82RN-Lzk$eP3>{%|4#lli4NAJ75x8Z6HoBTwAnorP;YG!V-Mq^}nHHMEI`R*%g zP6Sp+Jw~acz)|*jk-M7BZm+VK$XIV{Mnq*_Y|+9uEc?Y#-s0p%b zn6FCb@!dAn6YA|AX$cVIdnu*VdOp(RzaU>^N|X%~qHo8}sibrLO0c5scg^e0(Z8d; z`%N-i(vt7!MR#|nJ@gPoSv>>#exYNEqi{7+&JaoAODQ(ou_p~RVV+CvsVwmGqGu4_ z=F{VUv(;^JT!g z4n724;B36#6cElf(g-72fIsI+IS2f2!a1Ih3JjNcyx~*AO~$~%a4cQ`wD3p$_k~xx zfn#F>c|R)K_uqhO2FihA^F1-2xc+@A{pS}1{D}Osd;@MBNdV1VT0Qe{BZUP%Asr~l zhg%`t76VWbKF*(zL?FS_{q<>XyPGfhK5lpzpI;`1heHAVu&&sI^x$$_nh>Ve|Dh@*$q)w1#Er8ht4f}FURMTToky%&~`*u`IC!)3@0e% zH+7{1H?FJDn;4IeXdRgL*n7L`ThGm%-*3A3REq z({g|WNNTh7x-#llE9WV#&eY}BCVKz&K%=&*W`)u{Sz;Exmk;Dz{lTVkC-P>iRAMXK z;_()gN{G&y`}MnEJLy1sUZfp~Y}Y$l;e>T}F7)Bv*Nb~zFTk23nIhK> zXS*-W77nLuZE*Tr+68)^i{otqC9?6*^Z8HGc5PAZrVX+Jg8XUc5>)~|7QQc}Zhc#d zx!mGfTalfKV>#wM<(_U#tUaa@em#3`-#wHXSX{x*9#B{i4_Gq%^Jy@vO$9X*d5t=K zs&$GFUg{R_`Y)pEKJ2omnKos~Ojrq{x?FqTneqesty8CBmlodKKv!7eAEZy<*)KN- zF5J7V_Rs9r>&v9jtb;j{JY`YwmffmxxB=aYwoIy}KzBZG+c2O_%C^_8)JiP_(&1s_*3Z8uGT1Y<1BautSYTU_OOi7uJn%v%DEC9*Rw z3hQ{k_C)eH_Qm;M(4*~lvb9O_ zTYRxApD%+7cQS03Fsi9Hb9cDM+4}LOcJ({{uh)xDCQ;-c4B~)W;dp zSFeZbSc+Rc9W)%vPbZB<;z>3)MQdvJ7=N}D)jA_>Q!>q<^i`|3pe1LiP_<1a;@f-;UVIS;o zk(OzG*N*$XoWD3~?M+(|r%~H1gg`1Z;ZwpMnfkoVAL-p|!0?B~Fmt<;?FZ+dXtHvm ztX3LV@3;OTIz`j^3l5&34d5kI1?M%czyi!0OZZILwrgM}927pro z$JS%`5ZdfyI=90OOwJ$CPvlyNWC(1ZP}6H6PpI21vONsO%`A{oZv;iBGk2D zBG2nK&61Q1rIg`(w~D=e?7Wp?FolU@FWs3J5AnQ?Mr?7LJ<|HDM(!l?wN0;4|B(>D z_a1DdwPgqAkP8i2-@G1rOv|>1D{cQRj!Bv!O{wSUdnCz3o!0WA#h)kj`MRsKe@HNS zMtFu%4)3^xR+0?34U))_)^=FeH3Vxl$uH214c4Y&jxR@*@Hdnb>O~9uHua9p2Dl>* z?60dm%Cd8=G&rZT-p`)GZu4!tsw*$l!DOZ>Z~nNoExz8S!-O5S8~x?Xqlz((B;Wk( zC^_%Px%*(BHu>na6fQ{8RQ>n*&boNp){SxQ3jeB7DVT-Hm)MQra>MeaqGbKbj>MS9 z^tuJcYH!79{fi55g5~~86SZB&>~tfB-zPp*!i7TRvT+@gf^^M*Rl6svoiDm1+jChn ze)O>R>mkWc^C}dDv1`5CClnupw0`4R)}i^!*{fy#$86fA&KmIZm`iHAi{eczVb0@y zZZJ?tLObw?dQ3Z_=Jj%p$0Bvaa7MZK&5F1PEE5g(wtUo>E75@IjYD>!jYos5zu=iM zEvpLDO>$atYgL{)9ykR5f{lmGV-9o-sD+9~o ze&x^^^&2dwXkt+J&_|}K7w5Z~s#-r-`kiv~QRPd^PcYZwYFs6vKofl1;Z>x0SPd9-F&N1_A%cg<#-$cA41pG;-;q zZ2lMRN5-thrH{UUd#oc4ui2JrK_*>SEn8{*`jtltp-RdOYWOUMRKJ@o)nRF(go&R z8Z=k>CL+uiVR04rEtul#escPvy8lC^o~-sf>wr?C8Hbs}@gxaL_~mzM8ejkDP{=^} zc|UWZUFimdcT3eiGm-L}O6IF0S_+hgSZ79CGS8<7kB-a7zBO~S=OO6U^$+}>n9PgG z=$Uhcc8VTPH$Y?z=NoG{RSh%g!>e zD9o}%e}Vn%3taeK*tq3k1EODTw1}m<;KyPlSG1Tq69!?$JGbW%X+kDXF3T^##7kU< zay#E8YwuqC5SN}6d3G*by4KAwZS165_}q$gYrze@0P?Z?PR@7uD2y*${71+}Zhu|z zZ!5=WhZi1ThBImHxycmpWUQJ&3R|lhCgoLD%y2aDM#ldP1C>3xu83RfVs0{pr1cSr0dd?Oc>h7SZdnKm!Ec)4Bm`E0gT?o+g+WiGRG zCH3=__a%dO7Cv4(**R$M!e4L0V*>Z`9!;**)le~pFVALHbNWPvh~Sye$+5wt7EDxa z=2R1?lq6@`8kXNFR9suH`zm3Eoc$~>SXgIrcx9@!Z3GZ`De_ zdm(}vI;{TyoY>RarE_CnZs)Sy0XO>>cbZ_NP}UGrPmi|1?>>#Qf%TLc30Uptc^zdj zcECOPli{rt;HOuYN0!fds^rEsa#EnCWN zVs%1`HrL|XYC*XHWN00+*h1*T0cTV|?NfzhqVH@nWtMm~Fu9q#Lysn<$UUNMdU@YM znGB0Q#rx;7sj0AZe8o6%R z=<>ppr*(9c1KzONdtlT1x_6#T8&1(|bP7yr6^18mH};o$rFz~nN0K-fL*y2(2uoE? zZ{X5d?tN{eze)4zc_|$gg1Jgo>_ew3`g4{AL$PO8ykr~Ol*9L|+-;#sLFdBfykDU6 zlfbr-i?Eyn`_ewk1ZcvsDt2*W8=aQ}Ou^x66I7dxdF}9}MgRu@N{{atNF=@P=~5G% zV)3b+P^|VQ5Eq-_X76tDA{jf?wB026RiWLMk^AJwi}Den>Y=UCr9P`O1*R{zz>K6^ zr6p{7XC-ZAb9b%#dN74LpyZ9%XOR#xZ3!VNFYx+M9X;^ZdLV!_KsHYBK==F~#ybM& z{1+AX7ZvxfY4+bi#RaJ_ru`FtCiSsCIE(9xcSB(qYdLC1u4!(Zqjn46{VK#{eSs23 z*&_fox#IbObr;DG5N+`gxD{bb2Dc`%fE`d}2G61jMVKAVvoazuCI_FHmPh`<;vH+v-eeq!&Gz?RqXMDws zgg4YYsKi`=CtOuj0!r7)3p`abu6GpVir7GB1xtXz>#1x#4h^0p7?ESK0_&@|wO6tL zm!a~WirWBSYf^!lyKs+hW9*#S5F);ViY5P(iSNHw+yB@8heYcDu#SVpv=f4~k+cHx zfox=tOr!D+ErO~5@In=OfNJ9DJXi*n6~{pMW?C*Iko?tC7ObRVth@hfizPkcDqURw zmwQPJwr9sIf9&(;V4&x+=<_F`O24m zdzF$5jhgXEO`Nv4k~RJ%`NN_*bOHV1lw{9T^>TF)0@Go@mS*XA6$hws>-0~+BaVV? zw{LX4{nh#ZVXyqr`Kt8guhXB;3*A9rXF~Z4<)D%1t3Az-3S!WSrYH)%j$X8MK_RAMPOU0hk<;U~*huNCAmN{zJ3=8c`6G-5s2mH{D+o!YxK34k!3J zCW)p*#BZ-QJ2L;3r*EZSgc?TVAO-SN7c=ZY?7nuTubA?3b2_OxX#9fLQtch*DZ8bb z$!M+O@6lJxj~18{ZHumvIeDDxf&9Vu znzdQGp;~GEkHGra1AN*8)2sTS>uEq>E4T+ZoYp~7N!)6?VIU<4fi{7Wcf4`2^CA2e z5~x@Pnw2rB!P>t}?B#$-ol+^^(NO6s+Cy{~x8KgBF{9oo+IEwh=qlIa|6GabT}EX; zgT7Q#Or%YMD0+G9S9!krEge7sS=rAo$Aeja0QhkCUBcKMS!V%&zZCa39b{hkC`Jg9 zq(;+R>D1ts>Rny&x*ka|E6exXrT(i?Vo*Aa(vR@DK;k%sz5Cum?;s$(yJ<(M! zMN<7f_BZ0_b0QR?1O0-y?=rBlj^^SK>VETpo&_@mdprERdpT9!A*}I*0t&HA(}$laVfR_~Io{*c9GL`a7BG-O6iYt3@c#N8Zn3*Vj(I^IHz!X@ z3R7h#FsQ>~>Tsu7?Tei?957Y3Da+S9T9$o&7kbki?xSnv$X~&4*kOQ27pjxyzl99d z?F4H)PqVPsWO>RHJM1ZEIa_=WZLiD$dmr?_H}coFSEU)M;5SQiz-?rh9HAztZG{O| z7)6$r%(zsEW5v7m?Pj60LS(a0hR=^=O1Ar)!*S4-_R@4np14v!o$2H+;kQzU?)zO) zy<1Ma%!h}mDzmhz0a6T5@rImLJ-ZDQ151G1X&ue(SlvZKu!HTw|F8jyGTztrLc8cW zS{z^k$Q9>{f%UpY3>aJ`QUJeNcV~99OdMy*1*$shv0m+WU_fOG3OIBClE9cY;VZCm z5<{R<=fnvE(e+g9EmR{&ws%VQ&~Lx?_y5q{zl^JO_Ll$q#?^F^5bdxsgE~h7D!bQC z6S?v&e3}#@6pZjAaoE+-3Xz;7ZLCVr9I>E%>`n-dzuJl6&5B&^T_Ah(YSYFj2zTnc zbMY`MjX$QK)%pai*1L*kdg~KsnzsG>8OkkN)@LXdQw`XDH6YC-ILM^nc2!UHbCaeR z*5Dw+AY;^)D>y``ZP{6f;&JSlQ9th0SA^UcGwkYVtQgoJprp#@zjVd{QqnrT3H>z> zV)JvBEK(tY)OBret)hN@14R!#Hem|(g!bx3+CWJQf?aVjHD7;&T@mUHC_Uat2n@Jn zStUUeAO2!<{f}pJF~dByGN1wfC&L?QwCo3iaVTCmbLw9Qev}>9NHQ-e@_?zf3INa! zhUypiI*%G>xMRH6)MR{SXYE|4Px zWK9CWj{Ph*)-^bOe86wOucZ+QRc`@F`GCE1TdNJ!XkNcNP*&1FSm5Zzt&p%qaNW(X zxJE|(r`i1DjaF!IE9EIjj0G?oAgm)$OZV)C!fsV*fPe}e=z#hwpyF3R#oy-3-vcVJ zv*Cmw8B9YEVf7VZ40pBQClr>nRRCi-jKL}MOKuRJe)yoWbC7l66oU8K?Hr!kv&Gh0Op)h&IO&D z7n$j_eufJ5E0TV7c>;svPw-#toc<)!Sm5dWGu8%Ahv`6zHgYS$Z}!6S^4G8$1hpCw zL;&LYR%6DwfRjbfk_Fg3s4}<-tF(FuW1L90TW(~W2(T+FeROW)&gsw^kRhySEXNX7 zVm}@UE8}8V%kF~pWDzGcH-7j}K#2>~{qo}RxC4~g&pi_`3AB3gTt47oY@~+%e4)Z! zU_YJe*tM})_b>HS1l&qJ4UK(2DF9@X7Y3S(<+2gT_4t#|bZCLyPYLM@vakK*cLxIZ zzhIEX^Pdy|b|NLPQA|>Q4aSCB@vp(yu#5hjrvFn`MgW7`>B}RO@hxj`u7bfXoYgv$ Jb=vCo{{kNAJ^cUx literal 0 HcmV?d00001 diff --git a/tb_plugins/profiling/tb_plugin/docs/images/distributed_view.PNG b/tb_plugins/profiling/tb_plugin/docs/images/distributed_view.PNG new file mode 100644 index 0000000000000000000000000000000000000000..95bf38565b57be649b2174eefaa37586cf29fd4a GIT binary patch literal 87217 zcmeFZ2~<;O+b&FN)mIUzB2opRtphW0B*>IHQbmXeGLr}hs7wMfgg|0N96(5wd3FK> zgfNdum`~UT?v(A^b1UKyc4EK25*Zpkn zpSQM9*tm0}oSdA(nV(NykdxcMmXrHt`DixLjGU+6-{{yC(Z26Wr%xJ88ShZHq$DbVRqZhJ{vd?sXedYOA z>c<{?DQb5e)bE;lSsSTca`=YJK~_LT-Tle|(NkX|eyq*x5nJf`5#B@?@`3H}W4N|M;Vi8r|$4BX4zU2BUds$Wh7RV74mSmA*QhwaG;0xP&9yit*4or^qcFaxpB? zXG=DY1R$^v{N;`Z?L_O&3aOz86 z-G3;5`qBAUO(6?oR!psMSt&Vlrl>renItQ3lr&2fpYfo&0yuK7`WdFwmzM52Uwr|V zh6GEkLZ|ZuVDk6kz=)jp5yQ zW+Ge;G4pYuE0Cgq=4EJ>tew)xfrk?Zc6nA<4_ge|6R-!}vnJ{pe6u6yYse7d_N`@J z1XNOFmV3^AkQ&nHZZ)=nl}6mMl>q8N7z91|+x9E1R~Ba(np9hg@D&D(rHOxUDsjn+ z2w(iRBxjbv9wLe7p$>sz1IO<$6FG#uvH%H*_Hy1;)-%i3SYi@*4}S-xpTU<(zners z7JFw)qnbiTy@1igEna8laU-&+ zQKBjHtn1A|OMZ#WKn~&^WG@?K!n&!mp5e`5HQKqc^fp72aBA^<)I^ND0L?a8-_z)x zasEL`K=Tftp*AsE7Fd$iCsCmY-QhFKnVwNA;H*>aGR6RR9JQGDc0HoRsVs>vX_fg! zO7TsD6fEI)<)r{LznM+luR2l@yCRTM(k36A!}pZ4KI^7vD#tX@xt{v3Lo9r$i?rMFb8vJxil)?cg6T%%K5{V z9r&UPFz)+?G%D>i)Vc(hhCUgf)6>&6O_K=Ylg5z0-{yFHD+OE0pK3Kh3srl}fo*rR zqE=}01^QWBeKww5STY7*_6ubJ=LyTXPZwRL*Qk`3z#$*%xcTs10n^vUEz+fhdDohyZAkBKL8eZn?!q&7HIsF$zUlu^mjdkOcz1}rPZnma{AK*vt;y6vWO{IB8mAH zW2n~xb7#s+IGB#V|Mf(Hrs=Dz5BH<5;ivW}_XbGfW!}N4pQV=x?+yd*ITNbrF<%s+ zX=R*yUhCXIH}l)uWcNttebEo*gD}(LZX~-z23Oh+BeTn z3jIzBr8XlrQDppqpp3eUfTb4Aq9Q`HyhO~ikt5Q~3shGG2siQD`b0`otx{og6=19x zQ!817jHoJ9Ojqt2vV0({qJ^B}5N7&h$N1soDP~0vh#hgL1EM+q{P zjGX*70L^H2aYBIh?1E|yS@kHgDA%R#&JmH9w07L_JjXrJdcYKc)doxFWh<{q$RAb? zFi8J!)62y;f&Il;gVz}IiYrF3D|gNVhG0v2w6+GxJ}Sz%e%E;(_qUzQLxds(1t2c+^@N9 zQ@gqCo9kznFZUI(au0_$i`Vyraks)U!&cK3VwV*BuYrMsMN4T=mwnTA)Ul!fwC47H zn9MFzbb=>)V!pryzD`!N=Z@^HBjCcHHHv&7aJ;g4F+WnTKcDD$83y?t`LI^@lR7kEHnta#9; z1R2CDM@1^tXQV0+#U9JA=NA?mXCi=@D!FRZ37KO0rj zC}Tm7i0~2-1Wu9IWQ*}oLvp}%1D6ZbCLWSqG~c`;VaP5`oQas337?oj3YrCjjH@9N zMIjaen!0UP`l_|y8dzkcTLcG|uHJq1$@A2}-T7lf6L7G`4_Wn+U5;;HhJ{Ex9Mz{M_kLiAzc&R}jkWI!)` zoC7l|oL9q2?74f|o+Va{r~Wle-+OtYpc@GcL23 zypbVLN|w(2T0r~0>g7j%Q1c|X8Z-KzYOc@Sb1WhbX0C4<3yQkEaLE~euKu)zm5I)@ zE!7fJzA3he)>lyYBTRy>TtF||QB_fU{NcYf31WqzqeMJb0R0`whC0;AE{f2v#@t1V zR(oZTz@&MjN!g6;;yiE3t~veXTX1k8DyDuX^<`TndnPWl-6FaGoa8JiP}&#`hAid+ zScd9Gd7W6O%Zvu2ZC^%d7lmpIUzdA_Am;2RG@g_VO z0Y(wZq-|GFRXxI;x52TD-C-&zg2|=piTO9o5?!by3|FcubK$;i{m;q)vItu;Gus<} zLK}1=sG9>bVo+69q56C&VQJJ+e2OgiMf&nA@;et!$F1*eQ1U$Ki zcIWxmwBk7A4^M8Zs0nt%pt>_fp;m|&oPkX?J<=v9QK(u>`z|m)>OH}yCt5w zf$3gJ7Bpg_qXF}#fE)yk<5vm_I_DnJ@xF_p{KX~#99st@O8!+;P>u^)$?`f)bOnAZ zYt&w6Mmca3WvOl!oLg{1CAgwdk`D@>eq(L}BAmGmeyCE(TR--5pf>~%N?P8_dAz-b zOCe(60V47L#Q-Tv{rUm9#jppH@7N9|qBG z#CLB_P|FC_Wyii}+uxZLdP2l0^=atXHA2B1%>G)I7O5bXk zrYkVjbTk`?gmluN#Dulfi~gFUW{PAa+XN@fiQ#?gU?F z*OawHKvKS49~jrM(zlDWJz3DramhGuF2A?B;m6UpVbmxYS`FeYDA$(d&xA{P>t}sw zD7}CNDhnb+gFET~q;Z_Pr9503g>^A3)##3~I#WvJvs4Vst|I~iGJ;V^z3|M!Zw8Z@WatEOSD>%q_w2S*qyTQ zmVl&*qa+CpU?(Y)FXGISkX8rSA~^PAHYmC^6KXVMB4g|xER|4ZsdZAES#5J9!D+gl zj&DNqkUh-ga_SroD6%NhBw5hJfmG49zn1q8LtjmDsH$wq%IrL*?dzI?LrU;8N!GAa zgDfQv3zvC_He`kakw=oIQnJvM8V*a4q#b(_xB@)WoEK?WruTJ?ncN> zQK5~X_C=JxzzGQWS(qrIc~;5`4;!F~UB_JQsj7j6h)61_&u);lWK|ER0Ql6tu1rj> zV#Z=#D`?~(*-hvXQ#^Z!C@Yzj3Zw;2bz9>6WPHUEFj@l1q_D3h3y#l6NoBUsCJKwq zu#k~vsm-#wB;t04Mjg&!qbMUwrB>Da+VTs}f9Qc^L&qBzO>H$k}9CbT0_7xjOsHrGw`p<0wxt_Y}@R zld_OIsWJPM*A$K)U1rVlePkjAdipl6uol@5kOvxv$IRw!B^N3{FCjztCKfk6}w}#75)QA97-5!?c)q_6#bcj$rP(&(V%5KVa)#QA``( ze0RF?J{@268sGrV@rKq*2f7fXJug*j4;%?JC>nRj)P0+CIM7WB&HNFrrEA@z@>i4H zaNO}^({yD(xZ>1*44nCY56`?@O3t7vX4RH4sI zEl_7yr>3|kMUYvz6m6osd`oB~jeJv+$~_nHHMM6NU8}uF-=J}{L@tm`N`ff?u4Ep2B2BuPyopteK}Vlr;({*(_zhD<(=<3YE5d z=h1l|(jd}-4bVDV-DRrq=VjbXxC6Ke^Q)N?6(BU2EL>sZ$`kjDfOHLPz@5H2DJ8Gh z+*@#?e^=k!^cA)p7%liJaqr<|hwGHT)mtY|zMrs+cnZ_X$=?b$I0q@!oTnqWGtB^9 zq9U{6Hc#+Yp<2W@C)|Z^IVILc({@{Nf*fWQbU=QfUF_Wuxsv2sQuCkq$yY0Tliyq@ zVB~qtH}lWh##w%(xpa%(rr&-<&4EY~7+tp~(6(1W5x+iq_Mb?|msT81zk)S$cQEFx z)n?yR)toxK zm_tgr_p0E1o{WozNmlY?4Sy?o?!7ua?GG1?l6AH`wCc(ecjei^SGpbku`Wa7KqRZ? zkGtSfI^6#5-Mjy3MhYRiY3r7eWN{BoY{lzT4gC9j_xx^9!(>v)#|jH^z5G={L1hoLUCB@ zpUs3yh5wtU$PPoWd1N?N$`6#)j$r<-!&j9Xa*}>G%b0zB?i51Nv1no65+f_H)1gF# z`k{dO+Wz4JV}%unZaOLz3D`uluYnfV>ZiryiSJfp%_qo8zh9uha~bEyk(@5g654VC z?VhbY`!ESPu^dRn?GSwP#{+V5M5RW}Cou;Ht+b!v*1tTdGw;@Tqe$4MLio0@jCwUy z&{q4^r(540(;Eu}bf!M+q%8QFZuWVEp$v}t^?kjS^(tEI&-a%10%l?xKGcE#38DDu z#+#u9xb@kM=c%F;_=@tspT7s(dMIdUt@*!y^l?b<-pZCj-1<*Xw;0<}*Yv)1QeI9L ze4g3SryJRlVY!wdz_X)+x@WcjI)BKPjZuMw74N8;$Zq84XI>rx>=fW0|FY83=YO52 z_7O+@^8?Itfc>8RZx7(>Ml~5O2@4DJ4t^3zM23}ULY9y7I@m<|`m&2i_l9`UpnG<( zUpTxJwp`eE5&i&#x#8J7svyEe>07_BvT~`JQlGBq@NKAaKp$Ze?-P{sY2`N)fL;0JCup(VA}n|x`pk@9BdKq;cVw<$?4js+Ui}5m zV>-TlHWvP`F}SOvs!f>8>xG=H#SwDeQ{6)vRuYM z@;?w0BluL^J_k&!zrjkf`KQs|P2_)EsR8>GRw|vpK-z`jz#G*vgeEC9X1h^N3 zlcq>!i_mX1rlvG-_ITKQ_!EbcX$^T&C#F3onE;2{fubPR&gX+oO^3031GBAp#^8B;&Ax6A?da@x$zjJoZ$ zdP(jkQuIOxf6C3XYtD_0dfxbLw}12Y+QkUoGMgL%utw_0myvUA)}a$-P&+7j3|XwZ z2-t%lzsxWK$Dux*dPx!YAL=pBY8#t^9+@6v2`D^6>OJ|>SQr8aV#RVJX-R{gyqbmu zc?|LEaex&mj+6{Q4^JwM|P9{0rRlt+6<|F46$(PvQTR#{rl>969`9a$} zQg?8xD(E6q^46r-LiQ$P!~)+NrRJ}b!NQe93gtV(wxATr z2M0h2eUF7w`9UT36=E(1y7PfXhskh2_V(OiK?qOAknPI9HG#Eq)A;08mvVqW?lR-9 z!^TS|lNNV7YLwtFB`ozZ=Lt@*DXxF9{JGPGZ)A7#H>OBLJIL^DCwA_FK2!C;D zf?$JS1rY4alwoXBv#LIa5(#C$Y;iVrSs@_rgW!t8&*#QAO%L}Xi_e<1+@M%M-9P8B zp+t^)c}i_QF0K2DinMTPye0?%N**nXUZ|L5xkskeIbd8PizB5S4)F)6Tj=-i0{pX41!#`w-ASs*u{N(d3Lc=E*|1%{qm33@SmjRu;2*U-M|vNf z!x_yd%kENUi;AYO-1naDE+&)#$%O;u`g;m)!CV-=&4lwTiC5KSJ+QeG-e~dhvClHa zugdhQNjFlG#K^^Ejyl@X@gEW#N_-u1OL7~RUOj#lKB2;&vM66ScAo0;V-L#u8P4G% z7G=8a8_<#lGRu6H-!E?gso9T*{YbIk78LS2Fm3+1wc+kq$#;~%nlazJ>yB{Z!W+D0 z^6^iWb(0I^Prv+9far`WiZo8HXd@D+_XaM!@xMPwn`q?@Cb2g&;UD6?Lt<-`QUwL1 z2nn$+j7|U)c_AZc-t)TUDitQYH#(IfD0axjs>>5qNjGMmC1^s5YjM^^_v*4#bTC`< z=E8qa^NHfX$HN7CS{8zc@W_Hl2Rw_STl_g+ly3l%d}*};9q#}cLVYtAd;WFYKqnoK zweN#Sukc<$tsrg&(Q^Ut56(dd_H=Fu=JAcH$VK$OL=mf*XYzTq_?_$1)g>r!6ZP1< zQw^Ht`0?qsT4>i;k-BL4S+Iz+3g8LXNIIMDOC_=V24rB$Po#rbIa7V6DyE$mk{RROiyvF zJd2mE$5tk;bXpGD+Kz4LuSLy1oSj+PhGRE7njLKrkE7C(a`l$q5_1E~+MCRWqwcYW zfgrNdP*^O_t7c3+V0Yf{s^gasnN!iCro1v(9*1ydAO*Cq+F9YJt+Leq6$yy#B2hU#ZAx_QP1)}G=W@i-5ZOyo?g~db} z%EcP_o>Q~DrKHq1ZEDF9N+#il&5yEg)A1JQiN%5!X?)S!DfX4Pt0oZMN#1drGSTE+ z_50Jj#Mwo731%@GYLAiFNrV*SAdLH*;C?@VW8@2FG6h$(idi^(uE5R=OC7?R*_dsu zRZgV$%g11zE0DlmZILTPs%ot=wkOu2*rLs%H}^5%&G%FcZhO8aJ&f&QMpiZQA!a+V z1G@*6CY*2khR-~#EqBtGR?5?fPhXr0Y8Nx6LJm=q!A+|2XfowH9wzVC%nE~a$FlT&ttqgeTqNMXk{to9+;3;yQGJ)h$V3xKdLS4&BsJxtDY zY1MrHzX!5^VnDh7AE5M{T{jYfllq0u^ydeeN1d1KH&}B$^UI2$(1EI|%!3t`!5Zz% zkZ#e6M$uoaxayD|y9c`7@yHjo#z;eo0Jicvf41L8`1R{cIf1ZJ1Gh$JHwDsfnDL&x zVe(7nvxr~TemRMcBhW1Ln+HP0%W^UfRcpL_Ol&+BHk^o=T>E9IP&J^-&exoxcXdaS zfbMvfC?;lh1c!47WX^)eoaAi391~bLDG!$hISMic($~GUgl(tHAAhM%lPnIbG`yKJ&lpWg+@nJ?w7+7@Q3ujnHsDiNJ(z-58l>;8qM2vip=6+KZoZPD?u-m}lWlirn^~X%gY= zl7*jj&PHz~8$gx~jn^Z44zPCpMwYJ@`B%N>=!OI3FkY=s_#H9Yb>eE|lVm}-Y@Wu7 zL(qgnb;Z!ANhPcP0pKuQ`8mfSrR6_&0XT`@^Sc1!?ZTqnQz~x1CY#(y3Rk=~vvs`Y z8fcrqV4m!AU42pt!0&DOg|qpS)I7;Z-bD!C6Ebv8egnJ^F@bKcF5moVz;X!?W`W9D zC06qlYtF98VRYyCyP2p8H3xGy5Uuw^7qa3edEXfMlJK0EL|wzYn0ib#uFwYDSYp~tIiR(V2+GD zaT0oKq1~VfZKZcs6E&%sSqL2J!J_#-Xv1C>36)Fu%#e%$jL-1T-N-ZiZ4JjZlxF7+ z#nWqwJm7kTHB29p!p2+ey!9MO;X#;dK*ZeQaKGWYF;J>iKCWy}T3qO2`hd7~rJ8vM zOS&-iVf332vc97*yVG1U0|he+G?{BZnnxjf??kBMzB5poJo9rYI<{Jhu1*U#)XjTJ z#s$QAvkqE4EY<3!GpongWN5dfS0C(j!^Q5`Ed5rmwT-Qj$85@6?8Lsm(LEFOJ|ytL z1_!M31{|GU>&8B%LZ$Da_OfLS*_2Vf^3aBVPNVbGZ5K+-FRk`+T3i%4&i_3fUnx$j z9v~$XZAc`)Q$+!eH{ywEB;=IDo8D-1tW1mAlPVBmi2g%4qq3750;3Y9bANJF-8J?* zVBcEvl3|R*k$a$#Zt?z3=$|p_k_d=ts%H?O?u}j{^rq9chsR38;xV`$a)!bBUZ_SF z$&aYTPN-`JDhaq=h3r0u8iCdXv8@2WyfwF0g^|)>rjaH8$!HW60lH+jHoInBEdJEm znx#f-2lEnzAyD^K3a)kOu1qa$YL$ z@k6Oq?c62d^}L+?_<&PDZXxxt>OimbU8uU)j`>RseB6oIkc=^lb6`n29shmGV)s*O zpLDLzuq(fC-YP}%X%jbD07D44QU#94*Z}=qbU$zT{wxtSM_jj-fI*9=tRC$L=_#f1 z-hO5VBaDEW2aqAgYD*mANwP|AaX65+kKG4F?J9^xxMxIt=Ge+9fA|z}-v;P_rN$GJ zl>L25pR>igiKoD{d+26{Hd-H|g#&qIRGCx&-T>ES{TY0K+_P$epB5_TckmB!fBL;# zdDkD~fA*A?eI{fMwT5Ie*h}}FxmZ(IWeWtFw0O0-izX_9lJ@1`{shUjaG$!-<}ENa z&#^5mpNj)Kh82$Dx0XacsV7^}O31~rUZ8mgkvq~oa+vX0yuB{^s7sxe zEFtdph|nr1@EX3g;(=vQ3~x3%?>Vk6{3z|Tle!-%f+D^jGTQX0Y)Y0dq!Wc+P0S6d zXaAjqt-`|Sc*jm=w{Ow1p%Ya%?}7u)AP=WfXQuVs3OI682~XeJ9XVnMRPndo0J(pP zT?cN6YG`gfy6cBrTl9m97vx=y0et!o3`~E}ofxWh=JaYa<5~%KE;S9MEM~SRX&i?$5VOIvNlJI{$Bcl2i(uIv+6gFVimwg;u&G zai~!n<{tn6Nt3Oh&8`S2*chd$J-_yE%dkr6xkIn)h^=oGYU;3+#|(y!!Vp;fRt3m* zF!+e2 zGL{((v%ljT7gz-p10muSbIrt{YW!5Hpp#q(`X;f@JveVgTn|ti)B4GL_2++K*SmVl zaSoQT)sD}{G?Gc)-O<`HTb=c$@;T!dfD)LgJ7UP?z;~xkOp5U__12@_;d9YP%k;%O zW-@e=G|h&MXynptG9|1*Vm8b3y27m+yDr>;PO7E}Dm{1l%Nxf38$V>t`2i({F5rTn zoU3KuYR>+GAJ006V4EguY;g5ndPw-`3lwMt(EY1^``IfES5e@@KO?N%@a5H*^#zPf z`Xl6ij%hdlPZ=O`xEtx7_xE01u<=nvcc1~Dd@zPXrbh8`2i;?{?(DO@V*4VTfiGR zDGP)UyT91`4pj&0npKefQS^Q#?Yd2ay}bc7=-t*oZ{`>00KJAcvTslclwoXGc-SmA ztNXX3pd!4t$sqMuc6stDL|Q@(@WZh09YoGGDv4Zx5is7FJpjYodB4H6)sU_>m=2Kl z?1JY?#|ID!Sc_5VyXWk0TUEqPG5vZgF2_v^WZpw(hd~BD<%Ti-4ENl z9VpSe?|%XT3aM+Mvb=PFt-+Yt@t9Ea=mgwmd!;&z@Ez09v1t4{~J z2Ha40(*|vy>u0t~rxx?C&Zv=O^N)<0Da55ligb!8t+zc8*tkG8aPT{{6TU|7<`iJ5 zjsO<@D23XKw!byjk0qC`i_JfO&-uJ`^f2>0m3h|RHWn9AI#4?xxLodMbHO()p`-B> zci*`b0VB~XgFBZf;z-{oEf@4fI~AQY;vR1G7#oCrwO3T^5cIBy5L4= zP;=u@GPW|<&WReIUywXuOAafM=t{)4+d@!$@K+|3R^&h|h5kqK`44{poL{My!KXf`e^2EaSU zDIMjGvpSaE6^B!62ey|?R<1BEu(uWkGaj0#oeb-* z$<(6IM_GKL4PpMApO2j$sf_m&VSAcnAzAXWyq;0tCq`ylPG=r;&{H#-j11!r@DrU0 zN_~MphLAVOU2+2Cqx-3nZK(xLsMQoKZa_@-k_|9z{PG|wlCl-I6x^vsJOs~Rt zSc#zyJK-#>eJc!hSl7~f>c>mxlUo_xo<;A5UPGDZTw~rW=6mX3Sv!DpdliM-WYKz! z-e(%av`O3P`!Ha)XTP?h+j-2n#uSm7t7uZK(QBH3mXost62h0(VBfv*oYff}+~!lI z0g;hs-Lfe0Pu|@a=TNV!Mt82Q9N4cjPVc}T<8XKOTAr$!s166H^N{)|!|$j5cQ%k0&! zyA&;Vb1x9)UjF(i#Euo~vGY0oR5~6N>Ch6-AzS^RqYF*JR(YLp_Z`tJEgkhk1m4$t znICF<#;?3I*u>HgtUS#(P$4E~jauuK=hN56;`cIYt`=IQ3F0Q5D|%o~*WNNw`7liD^Hvq3llK$o#H+1697R?>e3d%odw|KI);M)}C z{BCkjW6+40>FiQD3(qUC(_+S7v7NSQ4v2H>z&XoKHhQ>R#YdDjYIf`El;tzWE;GdI ziAleCIH4X;1Oabw6-6AX_(*q}<$#b?$P;?*!ltF}FgqH^`0g(8QLPI0Vwgne+0)^# zSmvK;UwE`PLqhb=bLiS8H@x{TCUfoVg=b%1bdq*&bb0b?=&#C4iMG^|&{Npz(z0#a zyX{t(gF91OABl0Zo61fc0GZSVyTpVv);q8i$|$$(Ud!)D<{kvnuQMUQN$heM`+Lf! z^%(?dJKO1;adSXkhx9_LLDS}EEWO6lH4_K;V|nI$PyZjAFo-G3)=$wHS6yq)@K;`1m9PAfTnQ-kwFDz8=(U z{2*lKdgRCoYtYgsV9uc_gsD*L^3q{0gj=?wC3Pom8bfQwy7f-y&#g8hzwYUaoeMpf zN%-Icv|IcV51Usby^M4&^cC?zwE~h0WflEF~6F*H}LRK;LAm_*P$XibR z#(-=1c*ToDtJrf!>#>EoxWekMuICn@3lfe%Le#5qbqgQ0naY4Q5-c_gx>vqC8-Wi$ zFK$nc-X@6c-;)o8~0h3DWm-5dA!K^x4b19eLY#x7+<19mm1gZ$ z>IRNFe^t18vSLrtsk`3iL7SI4et8vjGqB#;2UC74E12^AWrFLnvB@3OE8N}q;E$fZ5E{|#P&4!6r(C(`3U;Q3-zPfkoniq1}Wm2DB4_i_V`W%TS z)0E1kL*bJjHyg)58IA8&>^#$E7EnL!PqZ6d%$U?Tw0S;f3?ZqvREs||=9_9Y)>U^j z6q1WMr7RewJN9Z{!|1AJb$3je3d4WpH0}wC#7nR7ZQ(_StuJ=gE6cTvjsYYB|2BTP z@r~(2_Q>qJ^@~SZw1G%qcxI=X-hjgP&FkD_JbtYm^4q=n>)!2}r`MFQ!ZIbd5i8~< zLEmy6*F2ybFNPZaG^*yAiJs)ZOkSCqIV!XuzLRV^i++FhK3gz={=J*pK206q&d8%r z1F^*WYG+s`a^^Z*l^xP_ASrCq^Br|NDC_G4=_#bXZ2m_}TnHUeLm zRd3YCE{V0SBv$33?1};c8W=mPrx8G|yJ8CJ^+l7lQJ^zS6&e_DQj&d&px5zV2_>zdX?BljucC`GDaFfnJH8fZR5N+D<*Wr(JEyIHz?1R<@K-Sq3=Iag- z!u$9KLIZd4$MiI65qtqf-ZIGI!P@)-6%%X~)=EKWVcMC=O50}UHsXFncFk(m1H28u z7Usrv(ULiyV~Hwo9QzDNSLnq(t_>PGjW>aF>WqJL3emZxFYnBoB5!0Mp7p8kV% z=ZIeQQL0t3_X2+9U18m%zinO@uwHllDV#OZ)mAjWKI<7Hn{x|hLuDo&ngf{#qMU$~ zz*CZ4Kf+s&%}3dXRzlPm>(~-`)T$2eT%AvwG7A>*U}4Q!6b%rL?MA?X)g812Hrsn9 zC4ysSfnAIF7q5{5I}k(~Byp$wHL`wh;ZfQC=j;)brpkLuKvmSf6s2 zw3)2iYht1y*W34Aqo!Y@bF%OLglMJOJEK)&dNgm#N+cJuTTbp$)nUP#8I+-NTVqga zRFrq)uVAyXgLuEDoQZTOQbU8?tl9bW7GGVWH{Udcm%WSF$2U0=eYkXZ=mB4>lAv2d z_N~u3TuOr@f$~`;ElU-=M(&cnpBn-Mnp232I%ZGxW#j{NT)(LqR&-^2;Rc~dmf-OV z=+r|+0OuKYy)bexzSeSB_4{-EH$Aib&L2>?)VUN>*TLsVl$j6KomCyVME`z-z(v1t z%?%`A-qgIRbK6wComf!6zI0K3q$Fj+g0W5>==jKwy~?Y1(6}D)HH{aUGrt>gcl6*T`?)phpF2tA z$E^-nZ%~lC$*PW}Wa^4t5u=*{Il3&uTpQN=XB z9p6Sc0ytw8IJC@e%G>-e06ci41*VoJ1(M}Cm*ShT+QCBH8Hs3~Q zeL^)YI@KD@%xCbgl5zl0#=Gi~f!f1;US!_PJFYN_wU?_upZ ziUH4Yg+KgB&9VM#Wb|=5(igD(W={}{V_j1<;KTmDcX%VY7(X{27RH~|E}Xj*h=Q!( z8nAsA5%>#0#ibhb@VJ!LH=ojOq%0?w3UtK(EYq^y0Y}>EZCS44*ygxZ1Z3iC_s%!b zhhS8D-ZTw&J}knk&Y1UTJSJ73{yy8G^k|u|`5|e*;UzwI(Q&GB?4tA=xp#X125@pM z>t4xwW)%FIq9W`aShH;;Q?o7MtdGC@qFvs%;1D+pMydE zwE%xe<4?%wh#sKVR`6%8J^3v~rRG{iRBc|5BGiopIb9RI(M9JqOQ6w}>0fA?NH9%D z-}Z%jljT=YziSepOqwmxvLw67oiI3oY z4cPTW*tVs%;1Lwm7Aci4a}UWLL3A?7G64YGNWeML&&Aly(7MOVsoL?ORk2l%BPc4Y zJ6`zHwSEm4-~eD#PV2xDqub9iRd8oivkNfVm0F)Vr)+pgz)J8)O)vJ%IJpDW7WAHY zE@)z+w^Pv80Z*0lf=yPnbQID-qJFu;J)GH0d?l?Z8W$RTD zHnJOHSa+0BokO_>HtAjDb>-eli}M|cZ0ArH(i{6J<2+KcK{v7oH*d_yZ?S!=)@O67 zby=cCLdI_dip%}Sz7VD1Pq6PuUIP|uYol_4C6l}iQ$*k|K`Uj0P>^M8JGmzqu|?k` zHFzwBOb_jc|AhaFUv=Yo40I|imQJ=Idbx;qVYnk%EH zMLXxf^F9G4G0_b2m3gKO0<_Dr7GrCli@cC393`(b=)=ptk-G^T*#5KFF|=^l6t7Kk zC@LS^1L~%K($MIm@p<6=`uFw$g|KHbqjAbL*;#x}07^&7BOWoO`pJ?w{!l#)c^@kS zxNqGq`Ct|qIU*csEGk#Ipuw!c*?os?M}<)ueKX5;4YvjS@VUe9~a^Tk25u?rIruB zm6Njv5ayp-`Jm0t85dg&e=XNi`$bS$jWVeKUH@l%0Ioj=xSOHhN}K=omr$5_|LqFx zL4xl8L+5sEcA-0vA$KPXQ!Qyd znnJ_umWp<&Pc1}sPvr?=5Bd&3nj@k$4~l`4xoWi2DT3oc40;;835X77XsM&$Rbb|P zIMA=U&snD_As*3AKRx{>h2zm`fLEV`sKYZ;dn?3jgZtdE=yTXH7KAX5F;V6g$fH@o z9fr>qkI2ax5fXIRv6^u0`LCw;;$*i4Pfqn> zcbEgCJFS#HaJQrAe1UQC_0ZsfQ`OSl@T_wwvyDq+Z(Qk$*WLc$j?m2mlZ5u#EanlL zFkeBI+q0MDbyuj#4YyoO%+zRGVjCV#?o8f`zhQ36P25&zRs|EHs<~5@UfO||BOB;( zK&Xia60};(C;CXSq+t2&w)D|Tdxz>(F#D!R?@q@B2d9Ja`4!bUwT{YZJ>-%jr;0pP z8m#vzv_A0{?|DtO%fip_P(bxCD|Cua=E0dPNn+pEjZ^_xv`2Zvn| zddlOOD_3*w`|dX;+&D+CF>B|*jQPov&6rbUy)$G3uaT$Y?-vVvw;fy#tViXz)dlCp z_`aueAjQdz4{!pW1`X%<{{6<~z<`Hx57q)+d-x zeLZsZJLP%KB>DRlgu-lN%DwVjSl2mf7S>h>fft~@IPQ%{! za7S$p>V7c{Zh&DyLVZ}7QXMT#a$)hn9tdXU0?hNe)pV*$xw%}+Bfwq~fpqZELGE63 z0*)u=<4H)#QrS%vIyURLoqetj+=q0hZKx9J>{2trp>BmbS~3eB4PSDhhoue&==8GO z3=*}$H93x#EJ=zRk(RH02duLU*vcI>9$EOxciZ7aRw&tbqNa!R%c+Wx)&AgnQrh(S zY-9kiinEa+17CeaJlBB}_^YdsGkux))Yi%FtQYp!echBNahjD!Vqd(}x=08ZN+2=d zo3T1`&F--d3GU>syRC9_ngTLF1j<4>~@| zbI558)na%N?E`$TTdN3)@vSvkzs4bchzW*ySK=HNfj24h9{?q@{io0ZjNrHy zNMfFVSb+bpel970VMf69&5NHzb}35@Hc|>4zd1c(a!=mxf$ty;wRyE^ZB!Uko|Z#{ z;cqZHQ8gxY)JskBLt`VQ{J`I4P-PQ6P=*a}nImIQ5fX(0Yd7fxDAX;UBR6X>aP^{=OWZF~QaxXlA}W>d_=8m)(5# zG0tK9a`+C#MliTG1U$NiWyPC>+mA{LyfWb^^$+Wg`UOrZt|czKBgz4S8oSJ)KGrrf z9X2D5Tw{}jUoni9Gv;R|FU>5OIS~RHawhN-3+5I0K4$90aXCkMmd38TkoNAYw#|Y% z(r<#r*u8j8dU}2k6hLKQU)6LAFl06OuWjxR`DsHrp-P{1Byh(RQYv0r3|@%W0(qTv#L zQx?Cx$8Q1ha&WFhx>wT$Eb}TfO}%k~VxSaQX2Db_?2~-;G^g1Gh$n)t)Y1Tpv+5LR z9J3F;7h8jmfF{o$TPOIrLt+8Gx8qTui9(p+4r-P{Ex4(7fqM0!6ql&=W9nX?4My?$Eqeu*xIa(gGFdx$V*c&I-A3PwHGh;*ReiCS#CjB&2xg zC8W@&oUc0PHPEy_6N$e~fxXwtl~dJcIWXbg?3VpfbeTDkut$%!9T?oB)+kfdoZBWS)nRgqEsR1gav)jG`hkggFcuM1%+k5fB0d z5~dI!BmqJQBqaG(K)bbX+q2L6o%cJx@0^`K{F(3!>sik|UhBH=B|QCREa^e5d4$0} z!XrrEcA}E7cCk%sJKcDOtIkM5rTH+Dgbt&tAr+RZR<(pDTNP$Nm{w?WgjpQnO-;D4%z$g=Gurr%?Y|H+s za}xTZ%6&H{`)A$C@O*euK>ECXjoP)i11(zd8ib@Kg3mfmeY?L-E~eNdlr58XtBsU3 zYgRak)5p?un&};>S2&#POhjAX>@6B@1(1tblj9xa9R!7_1DCR9vhFEVYbT5h?p;SLv&rr2Sj<0++49`F+(X+hJ7uWvwt$ zQw~${ahVg3Y~gQjncH>xD;BOT=$IxIZ9ZwB+~J(QC`MN*DyuT4sGen$N5g5WUnYA} zb8=`gNrDTV$>Em1;3<*r6?>6~&VWWMur1Qr$L= zf*jTKpt!oQL# zv>xCcU49u(FMzuPP3dTNt^9Njg2*xua4%b5H~|{>@Qt zk9?9!p$=Bdp%Nw2dWL)G>7t0Rp!beoDNiHi9BE#M%=AGWKU1hPT?1^f*S%3={eI@N z@9lM3@5Yl7T+|jS&+7R`^n@7H9mt$BHl|ux-Z5RpG7_}$XXG`hLisrN%LJhu#xJXpJen<2}meDx^@>B(9)L6qpi=;y%j$Is~w920FT+nU~mr2W< zaW_j}4CO8I#iln*QWf$KZP5S@tEJ-(J&0dZPfHx?elz83W5sWzN0@oJ`_Z6rt73_d z+jY+%eQ7>{mwRNjL*tTz78kH>WtHiT#hA;G(~B&hS%Xt2DkO~y=8O0*WQy3T_hzU8 zI(c=+7UvXWkPCxaGX0QesdNhB4$5OJr<|}jDPN)IhzxVV!RZ%Ez3^vJO!kaCRi2h* zh_b~;<&{4Yi(1ce`TKVQl5$1I57I0ibXQmkm~;*5a#jKb3<1c}@1dpc+HtEALr@J8 z#gcN^KgPr0gujXCURJCnto^-p-J6~f!(T)e!B$Nma`6Pk*h_c&Bq00nUB(u~`~zG5 zBBJY7C2Ml54Ga*VgyT+uUqNqktGBy5IwPP-Zf=x+pOh*TI6fe2xJns^in~W7qDiRU z9V=iu!#@c$i`L8KiDCe0v0fy|8v{XJP3iGf_|dS@7?-v6fsJcjZ`y_AS!p#IT@B|P zYYp#A8Ga#Asxm~>_cFJ}AJMFFqFc(yw0;`Odi{jmpXjmXLii(4a(DD@LHLfo=RsWx z3YNSt{ST3YulQ7z@8;A8Fea-K=%o4g_~ggHyS@EnhND_^oP3l#GYb_?IfkFhQ)qSv zFyz$x@0(!SA9sy-Ta=Ars1*X=moRNIoFCvA(u!igwI+HoIkejsn&~fVtHl@ zB(Zxn@oo~!t?;xlx~Fx7T(5_EmSL=Z{>Fj_3`7W7de2NviDCKu{H(oPfSm?>5*#~s zJpI1^>mzws=ftK9x%oB;HRl_(Hg~23I9QFGT2*CzSjop7kG0dt_+BS+&%n=t_A}$b zEpxYLF7!h8Fk(qL3av&Ox$9fMfhG+y!}c!l(u5IylcvYC_9n6Thszm+foitP zxLoy;^|uk_=V~>{`3{Cgg?bH_w)%w`1HLjP6pyY&4kJe^4t2XDV@L4-?`$u=BGh%I zSE;$?q7mttlo|Rd8JQEy&%()1Sb4E|>$=1x)l`0@aiczgo@d>C?Vcii=GBF>SawVZ z%bxM42&< zLUStY$A1`@W&+=Wtt)w>1PB)$4FFvS?HQmZSsf1+b1`Y0QC2xXZI(8MF;6uAC}2b#=PwNK?p3z(zeHX!DK&#J|ejqIV(>~3fLm~3G@{6 zfG?P&ag6&%1Nfvtz*TAz7e^Ldgzpakxu1qU(H*x8qwS zO&$IC!UV;&A>%w}$@F}CSx_7u6?!s_#&(J5_mNBoqxy0m>Nw4NCTUb`jYR|7JN-F* zZTDrTKJO;m=vU{}f83{z@=gg>=LA2k$o(FQ?ed_E!w=e5sOOe9H(c(24AR;r(0RM$ zZFmR~5ssQU8QHNo@6XIrdbtTNnv&rs)9$?GW zpSNn7%Jr=26hA;BACezb&A56l%wF=}laL#?$1%Mx=<5Q>8}crzr`4LG@~ys}$eu@K zj{fO&WtE&F)@X!zLVvHwjjXOQxi3Y>wQ!)zLY^4<<0YRbm8)G(YNeCjW52EUCgmsL zA*{tpuM15|$xxrFAqZ%#^4U~hyKQOyQ|>;`VAC4VByv-F(`Ea;!4Gzp+HX5e@rbeU z?}@}+v`Wwqxy!tAzBW&*bThpyOm)0cxWU0hONHs@;BPQ~(L8}2zzPu#tHVNfRUF!r z=t_=g!|u+eXCnnAJ1fuhxr>cjp9Y#IYbeMm`y16Pc)-o}(QYwx?pArOmMPl%$*PC5 z0}t@qtRin;FFgA?Ww+1zHQ@{TR_M}KP<7w+x6iedbeN)TlpgSp??hkvQcEiZ4AlSY z{pwB5f%IZ)KhM#e;9Z5aWAp6?9+2nOHN(fdt!_(<6`V@HbE-*jjS09&{N?&f0%?G2x^r2#mn@hJ<$p8m`cKlUO(Oq%xCaNADV)ZD=>ikb86V^T zVob@SqVmc6UAEp8Sj_8nu$kP>^FA0TjvhXFxp=Qh0~-z&3%aeteZ|}8%}+f-u7mcWr&M^GPH$cs2Ra!z2}ls)AZQr@p+o)nuRV_l#a& z*A$-II0M}W!|WCBOR*a}btC!=OftydyDJ@rAzeo0Iwv{ZYzt9v%I=-oB4k-Y;ORpb z%{9sU?mt*)dW!&|Xd*ECf41tjWJtk3|1VgUQRa$B#os$9-Ysnm|A#Pr;#tB4JeDu5 z!98u4S37>VL(_%?m0U`D7ePOi^x|};-smk9_gZjnpYhR4Jvx~CBZkx>67%NAJ`*o1 zC>b+Q-lHiU?I8||r5rg>kt=`xmX?K2Mg!U87JA_y{ihE^2Q!0> zH?D#E3$n1}Rt~1axJAo52layXI%LRfAuefE#ZOWa7N}IRH83#9o>Y0gO1lB91e)Aj z>MYcu+uhvii-zh4f-1^%2Q81p-CatPrD?J+tF}rhpV^;MKH<0nXBY-zCzn{uc7(SDfa~v^r@ILq2LRiQ4asGXAZ%DFI((ML9^E#Aove+r7 z^=ry1nSn)bn{eMLYur@uUNZeDekwRZA9e_*K2XkW3V;@9aLTy?wZ+ZoRF|u#qla0q z55o3j*fs*pKO>3@#PAdQ= zrt`)1Vdup1HxiZQub~)kFo8cV_-=pjWsc<1c>#5ulqPzuZ;gy|fj2;vcMsGLIjU2h zs*MMm;?HI%DKKZ^dsxkp!{OfHoSq1S`q^7wKqb>SdPpxh`uz4H8!64IU9L4zA_#f* zx&pAsui-P4vi7dscIoJ$8dVSHU{1i6vA41(D91W{2C92n{bV*r9a4}K>58+5rWn0J zgZglo7E1fOzz(H)64=3}dpoPU5sI)hIJbE1naOK@^)&y^tig}lG_L&U_Ztm}Q z*}(rICT$1A3-so4zCW@P0|h|w(=uEms!_>m-T9E{hxC`u+a|pZDxXmH@(oF+%ib1p z^&kJ*y)Gfq(qaMzQFZi;f9hDc*DnFRLN~C(q@zI^dO1I@N-E^st-}Qv-`D`*=^MFX4N9K#wV9cjg?qm6R!#_0pEU z(I>~=9ISig2(?yU?cv#?J+ytm%m4D>q7-1ylFSeIT-$zF&PUZVs@T!paEXozQ?(1k z*VVp19(bo`wk@Fk=s_8GpV>p|qPrS9@ia@xF%7spL^K&Q`ivx21}v49cc7!`bgVA5 zEjIazY0WMF`+lf&$;u|=5+<0V?7o%d5=QIixXkliK}~#w$U93dY}<8F!KnTfR{;j@7!F@mMhtk{@Utj)v~oB)NT% zcAnx9=W^|Wvl56yQoqxpnr~bG&@DRm93eK~Y=zD?>pSC#vGq3-kiy(Z#nLhRj*@8C zU2(fjasrCN4u>f}=xAmhthSIzD>pUim`H~RT5aI16;>7m2yrtcCm=gO=M9>1E1YvJj$*~rPpbbkc+N=h2>U#F#zHNnJ%J2 zB44I8UTUfvqMu%GiyE(7m*neG@m1UU64e5g{4@diCCizf^()yU-t!tw+A zxX1v-`ViXzYVX0iC%0B~4C=E6uIYT`uns@h>=s>g>{EDk=Q_b08s4&fq%V9E@%>)P z`ux$Ajr6Plx5Lj3`&#`{Wzv!wO|Bi`42np%T&bn0uB$mh7U@?X+$3WQ09UI=jjJok zEHKLHHBBVVsm)hkjpDpxcUEcQcPUGo-Xck?l(Uq+F`t52B z*~Yo_+e+Wi7e<8-YZNxpaf3)HIGQR2N3mDPi*J)f+4jEkr5=^7Q|?Zgi|!yB_PPyX ze}(2}r$dh&kULaE{hG4150m6_Y`FOO;fnh-3WkzqKlf8hy2~yVl(FyOeLn61*xYX7 z26}ZHNBx@~u^L0?fDc&JVgoz*F?N-)eZIS4_anfWPyg&!_xYfszzqNn+V8bgGI{Ti zi-qyXRWi;XUH({4vJ`xL%9@n^@Cm`mtNJh3qt7KG0u4^K4uhtH$P-J~0snXzd=fQ+ z@6LZ$+u|JPScA1T5VNl(W`H0?I%PNi8I`?8sVHc)f|AdDS%eDa%w zUka!1w*Xr9Y2f^Fmu~)?7TX%sVl&Z?m4uz_)@_^TQ-+hZ;`VJK?bJ$80W07ZxbOI& zeW9!#!^Sf%PwYD~GrQRn?QYmrIT`Z6WQC0G+D~SxXvNGc`)9ka(68gUvofX?qqqFB z+b=lS8+NL_ZgC{vt)h25!zwnXmom;LnhaYB-EJcRr|&a*BrT}QJ0DSfQQe%`X490VPf`Sa*Qh0Q9%#UZrllDD#dq)8^N z2{wD5KlAg2J?>9SY|pVz?Ic)Rdp3ES-__VWa=y*0G9AEz1z~IYr*fm#wq9z?As?Y= zG`V}gbJKK;GcW4$^Pgzz-VAfPfgUa%AqTRVwZ)37a7B(4w2t=~lAk@hmEA)!rUg6+ z4lo*|5bH;obnf4$j(2Uk-0wBGR~^bXt+{joeRrP&+Xas$`?yaX##J4p+@s?j&7BMY zC5CubkGBGYoaTUc3@MPJW5LCle##36RCq=XvGdkAQ`2KoY{>Rgnqe31`9qz14_fb4 zN%1+&W_UE~a6LMthu#b?KC9hieIegz?oE7)cbuL#sLF4N3yegR!Li!s-}4 zD-6<+(kNqQpoh>kapAkCHXR`bTA$mGG-(3e3SR-#`phSoLe%=-6dMA1X3zb!50>Ky z7;n2QsSeJLcu!tO^ZCK4;8L)kP;=bVzPW+maHknos}w>71QSc|Q_-@#*+Y98j?kjh zvkQY4@gYR(#08)edY=m0mrXM^?-Yuvzo%sVs*seak-HgfM#m+y%1nzdL-n6Mext2E z*Li$Zjbo|L^lT+4hI4&6xUh+i^P(-_R|{i>$&^;-%!MNn?*k?(sU(7!wfw;SGVU2jL*ft$IMoC6AfNg@T2J^S(K$ znYU3Q|Ml@Y6v>lF%Eo5ms|)v=$-UmhpB+IdLItwIXV2vI_r!>OTALe~MVU?*cFGb{ z^j4q+QE!p@T5V?Ysv6m(IhbSK#S7H6GaO=;`j?=z5`^o&<24b*Sj?sRLhPGRc1p=W zJOj2F4;}+N4ngZW-U83Zth~*7k=8y9dKTJ+Bx>16b8+Rd|D}+ zmNX^aNWG~5=CAA6yrl4VTF%mc@=+hSId8MDipJF&1`Jhr#$|t6r~N4h4-DpS`9f1g z`dCn;Hn~(!_sfVgy0@wlrqYn4G`_TiW0gs>-qjj@l68cV3s#4Id8%wHRLtS0R$$;e z3B7>h?w6K?t|zw*q|Hq{JqY+sg-#aD^kn?5@%foqMn{sAZagdL~h_F!kRw>rnw!nQIsq}|&*Y4&{MUX8@-o)ElRHs>^SEaqty z8(s#$hO3$7)*(uY+#AiYt=Ss%`k$R5jJi>V_{00sT^49g8yFbjLnwxVitLAk?in># z6!+3^dmkV6`~XgVa?}4Vr11CEK2=-ue={`y&CvWeL-XGZ%|Bsi8d`&uw53KE7UiK| zsa;74m$D|OhL(&l;MrewTM;D1E0#a@*Ka<@h8FZn36z@dJ8@FFCMfa-f2?qtcRKrf z_x{#hjBh4GU!Md|-(s=vBm0s>KJPbCoN(z0^ukk1))Sd-91LG&~ zorRGsPng3V;%7%HgDx?56f<67O6{Ctv_N4)w=tjHp9k1m$h_*H@%AgAz1drTkl1{< zc+nbpQ-Ds5I<@9S09TQoevPvopTC<^I~T#OVjQo>?Pq-lpeXP5KFNFjkFGQDCoS)& zlwihh?D2b}5T7Rtb*PtZRDMf-(&6S!%vFFntZNd@Uz`>q!}du0kSFGZlD@E`dWa&( zFnAuX!o$Ya$jdfHISb$A?f3ma6FodHyu2YPInAz_n7aBTA#;{o(cD*QeOT<|HV=O@ z%h3;x>jvfD49+*|oXt8t;U!A8;JJJt!fs;q5OiMfL^{BOm6DSw2Vd>#mo1M@7s&)F zFP@18gx16B56YADV3)^B~(#LRnCUoy1 zdl2YppZQo5-rr5Ev!QyP_M1kUh_lmcrD$LHXcPSUTUNsmO+w(466goO zuX@D)3mHG-3Idkg_t0Y9V@1?MAC%rXYP9#Xfmku!6k(*A{m+oCt<>v1Wf>K0<9Oi@S$ zs1Y^V4_PNWm8!%J;$9n2c?hJ(>wc}sQmw}98vEAV3!K{&8uHv*cBRX0VZP5-rOI2* zM5mkI;9rPlwG@^v9`#2-%{G1Aj2bzIH8!m5Qf_5?b|jvvYQF%Hh?eG?G03%|dtUIwC9a-+rVY^X*8|5GE zWyXQSr$3^0L(`<->4u{FUEXdQvCY2ak6L3<4x4+NJ&-xK%(tKTSR`9W~z7Y|5BXb z7oQeCzDaT-z%Rb9l}S7~WPhd?iVgL_-1iNfKpH|~)-=2E2bZhDJx8G?B2B-MC93B7 zn2`GNrhT}~vX)zEZM-jJ21Y7sz}f~_GJL6hGWs{y>SK+y+r7@NYwt?&etqm^&(Q0u z&PN_hOlYNk(f9u7RpmazE*Y1q*088Dy@taC!@|+Xo6)(kqg&z+s6-vo5o}Emzj?5M z#>tUnJrLz9En#l5{;>O~_W*V~LI$wgh6~D!v4%4xkNaO&o~$IaN>;dVriH?OLBUd*6w?RSnG*;I)K*<9!YzGL(pqZbP#1!Ry zPm!}LcJ-Ok+V{_C$Mw`HO6&JD1T||%SBPf+3`{G$6WRx8=2d^h(FDo5L9-!1q>KP6 z)q?Xfh}*!vF1+V>p*qIpaCk|u@W8{FU_$yd4qg$TD`XeZ+!l&j7T_}-B}JP;Uvp$U z&UnoVO&>Y=O!W179$Kie@LzYIcwxEd87QMP_H$0XBYS#n_KFVwt(vq3FaJkMSBx$i z*z4K+>iNe4?V&j?I=X5&O=BDoD+ea)O&rZd=e%+=ZPF*@Z&qFBEDUGHgXCj(ijW=_ z--pE6(KhBxXO-AvCXM_T^eL@l)apL73y0TonI5t{>`r2mZSFq8W`+t%V!v2pdXYai z)c2<-nzS^a$o#*GqOHBH7^Y~Vh14+#bd#ueDT%#O?A0Va z=iOg8g1(xm&S$AH^(Eu&PG|4by)6uV9kQCwqT9a<+qd@CxHfEw!`MbCrd2g^i3B1fsBs`2Zx7v|nR3%}9_TXyt}n zAh3p?JbCwOX)$2Bv|>>axr(n zlX2JhtaR}IW-NE|OA8+1rZN!$ek1USvpXc`us`>dv_6 zPrxg-P3K0$-2PdLlH|Vub&NlsP5aUkzXsy8dq;pRfKbeS-@GS^r}N*=E(9=+taLM= zC1L+}Vf6nQh0zto|9Kz0oIhz?6;ZePL!ROc2j;-xBZnKXr=@#A{kui@zvtlJa`3O~ zf`j7kDSkCrWCO%xyi~oqFH3p<(x#r5!u+NFURv{cxK#S5Z6E=$?d_eR%FDfQ*K&HH z=~^Xd0lbB}iGezL8F!g>o8O?3xi#*VqhRV7uTB@e#BY)^kcOHAfs%r&D1)kp?o$pI zsGhgs^)f+wxkhlK^){J5gmVCv-P6-pV#niw2m~BCPc;E7aNa9K91=c> zzEzy$QCyi68+h0i=an`Bc7@43@EZt|z(NFQTm!a@vZvcXW za!Cd#@m| zuX$f0zjTGcC`b*#3coEgJFDrk~o$jojJw@qz-iV7Y?9ht$bSm4sLi^~3 z{ok)RaPXUojjwJP>K2=&*jHR>mY?4pwyE*%&I=Vsy3}!qjA|4kjWbR`a5z*w1{rPF zsn(W`$JF@J*vaF={2;p=X^k-PiAXla{YJ3&9n6ARj7OiHN2|L~UbUvk7oPS+OpJxg zMy`xL%6S*YiPfRni>BlyQ)6K#FcW1Hfzrtf_>~{EWAj!{0S|V}7uQxhatW-WUG0A^ z_hYE+3p0P(#aY|K2A#55v@%5M!?myB57Y-XvH|Gy@&NPu8dig3stBe01djPBLjc(q zQcT^?cF41YKu-5OHKrEpX$FYH(A;lLX1>L7N-y%jQh?0S@YnF8cTdIg&yQ33d?fYr z+o_%~Q!S%zgT<7wqamASuiv{0dcd_Ra%|e8EGwl_=QSG_>IFOEyM^k9qsaPE%V$6)M4>Rk&MThix6kHD$*eUj#9$jCxX-y!!d@Y3h`7 z&|}3hQ{U7YVN~Fz36`*;3K#vZpV&#zC2iu^WT5Z{q2Txdb)IAq${o~Sm!SwBs*IlS>Q$$I*pax zl-nJ~*)S`q*8^v~EFL2mNI}S#HXAuBOgC^?mdmC>ZCXkN_6xlQ=Odbz(?rSC&4bXTTs%{u5VCTfRqO zL!G>dVV?9X$LnL2uluC~C^Gz)&RxE+jLgGB;3a*xH1wsP0(XH41S0j}@6N6N9RvUO z&HszjUimx;h79LDl1E(SQTccblt=3Xl?gI3?-;skfJiP#jT;aS?hCoS1f~<`B5vH= zca2^8=)fUxP}HzG_*ZeKR7HHS{MQqM)D=ZSJF;)NTE91@5Y0A0JQg`Oar2n88a$N- zM~GmGvmP(^_fV$8c+64)HAt^)_=objmW~FTU*G4lz9{OriD7i8oKBeo)kaG6H8r|s#iS}u6Jt{Oql-V<(9XH3D<^Li1ywj zP$Ll%Aq6SlC~}=x3~|bNKgFP(L?XhHMJoqI!p1Qnl*X9opo$PJ#Apsiyjbh~zJ@c) z4y|iej!f&5obM;B=E&)y=IbEPemDo0+WcHxV^AQFbfRrQQ*oJ>o5wQ9aTy)Kq#hko z6ozD}L)wk@&D6<}yh9t1Li)KaY$l{of+}Z+)91@ZJ$JF0iM<5#=yar+CpQCxevf{M~M1f(%MCsR>1a{}SR z6UD|Wrg6e-F8i%>ap5wDn!%07 z$Tfv+Keg2!dp9*BU?d|6{oXhcc62d?G!A1=F**!5wH`PMf*gD*i2sOPg1JY*7^9s` zNCR*cM`S3nrqELmJkGMoG~>WoNFNc2*=k@TYUWIEkpr^GEYX9`i5fh!#?}|fO=Y5v zKnnyM436c^%B4iV7k5e}b@SCsjt!~q2FXE4awU?!u3}=TeLRCSZsnrCufO0F?=`&p)I5Yo5hPdFoS! zk?v87!I+Mr_fQt>MOEKKNiTOy<;O|L+5DZ19+NkE^L-Q5Ws-gg^a~vG!pT-ozl)NP zQ(A#jqV+y8vrGZl%m_zmj=>j(8ZjkQG4^Fk@gv;TCef!5{Y-9*FM%a;5e7mK<^iZe-<%fyHd-NMaX^+yVv0?}=wPE| zUlF>&pp)3i2&x9J0kIEgeG)p{r`MR|fQD;Q;6BOXP(HQ@ZmciB&g)&~rj;5LNcw{2 zBg;ZF%15}NPU3;^NN=uTg^`5HC}Y%AO^#H}O$D@f?CGR;NsJ#17OBl^@R-|6=6}gWsJLKtf~N`vDdH84gN!)&Z9Z~x z&t$*3nZuLomMet8F9t^WPRLxHGK!VE5kf9T?JpA`AFsT7caTptKvVhq7yDGm!F~=; zU>Ifu-?y}uzB4Bl9hietWqII81_Lsz(a0AZcZPRS=b~VaTOq)hBQ7vG{96OnHZYol z-$J>7+yxVkyhjw`oNBl|`4W+~@1+j0gcp2`YO9V%uaj4ow~>8Fn@PmvQ>WuX%Lwhy z@y`SorB34nN6(GFSRU?k(FiqZBun+*>msP)&xN~k7mFo*$a!UWq;VsIQ@Pj{%*2C2 zI4WaZ+<`J$pr|h*2s+G|XyLNOXtNtM7{_xpX;4UmM~GG@9jZi*lUXPS%S>&2Yr&jA z8`dEa;Lz}$3;0l{9BXcPtw=+VT&R^7itp#C<_l8B-~}jegh~YQay1NXg2AP=h1=!? z(|m;rM3#hx?c}~j)c;e|n_~DO%D;NW$}9NFYs*>?-1Tk_IP zve>aQ*-Fyi%XPjQd@(eu5@qy7cE^40wzQBQ+4P&_r=j8F9#*uDLNvrNXEtITO3bvv z&`7P6?FXuT%U%iVMR`KF38j{mw1z+)<#Hs{+TfGM7?gqh6Y@6;s!UVzC2#UE#zYhs z$EP|FM9otLPA|~@@3w+ie&PrT6fUVT8MkP{sfj^$m9s0k_|DYARTpxgu#MZ+w(wmh zfjOgQK`~B(G7+rmcJK;pHPjYH)^8w$RjbNV`wD?)f^7(k(F^Gr?6>jTuMz%m)_;%6Nz;)u88R zw^xwJVcDyGyX;kUjK89&40y^vVd;LQZ}@AEiL;{eB+CL=Zf#&2rg~N0c*+g&ESXMF zxN6&qcVgZ|W)~rD+@sUgn~>DvF}~sr)%%F9bDHXR>AKGPc;jepH@|{iXeKmbVX;(? zjTi>8z1iEu-S0ww@I*h-ppe6+WT7R8&_U#Hs#rRN3LN$nM=&|RHK@9|w=d*DBU))O z@;LdDaC$00zm0+mdLa!Sv<(Lwfx}`_*`RP`VCD$P4ntxdC&4I;TjPv%nev}FEM6QurU@4aU&yK3F}SUv_)^h> zf{B5Ep#9Z?z6sA8vf04pS1Q*r#3IJMe(v;i;9w&o%ODdjOZ@IQsPkM7N6Kn>S1XWm z6row9h*@0qwzl}Tz|#^!&9HTH<7_$k6ots+`7~2`Co@jpBsI&Da7?Bm48sl?LpcRF zttOt66X=zZ3sC7DoI^&&<%yB;d=qhFtvAJqIOlmEMhi~g(ZWzu67B8fc5{t4(nE4&mKutDO?{X($5yS*EY&D+>hs$O_5VLI&=~ATQ z1&N2Q2h+jEv+`17VRt95&O*aAzg!r&ryB>F(EfIFIsep+pNI+w-d`nv=u>%xp*%Tb zJBgF%V(>+t-na+2y$rEo$%w^#$WIwIB8@3_0qn6&&O*|V^nj%?IVLP$uJZZB%Nbdd z`Pxo6O-POCL>Y9i1IO$i{WLj|o+bVK7gM0+lmDwJ^M|wleRC;Pu=M%glJGxph<9>G zHw-EjU40cuwj*0u*s8G#jY9H_9t95v1@<;JmUr?Py)bnDMJJTAu-qNm)ylnw=8VpE z*hu<(3N3VUR*^y9HJO34?#hZ)3-9L2+{UNZ5ly8(Jzj<^yyWKD`hI{~tONFi*lW)K}H*%6kjt<>ck{?Ov>G ziRLmV4bg4K#!dxquQrklu3(kVMgYB&QJ%;j-oY!5zcs8J7X!W@RsC?)xjtSeZw>~H zmtU9ZEj-fp)9}dbz)SzIf$ETk@J`t$Hzk)N!Oyj`Q!~O{BEE?_4GoK`)+;gu zkgLH7Bf}GpnWWic6OG&#^5^B{FZ&Xn8J}RWE`cYBYA$lGwCnjLXnCF#mQk%e@On6} zmO4!k7v3S@_52)BK0!yT_0eqveP1x+BT=)qGw9anmFZU=TXoV{0*8zU3r+dukM-!QBrKA@L|->ub1%AV_RRt70j_oH^WHJc2S#$ z$I-?csrZmSp%!u^w1WSY<^(KqG$Kri;?I#MNaDdS;A9#5)yr=x;TYDv?KM37SH9P^ zA)MjzrQ*X#k8dq4tlMwG>O0q2+IWD zrx_++OJ_K52sb9ae3L`WMC`@JvMTvqM1mTY@YY{C!DpA>PHou)NHYt&J?y11yRWJ=g4b!+vTW2eV~ziZaukKIo~E)W znE#1Wq@(`7PeT6>P}z_1iS0;WU~oi0iM~fDtJr~s#`vW9SQzM4RaA=|h0Jn?Jf^%C zVeV~nZ#b02>rX>gGfguuM-;8=HAy|8$1wMh^WOYH98* z{>)dL3x}tn#!I>)RXk^pWuIY|A2*U>S7^Yh5B}-WQA{HKBFv;U@9@`7DlO|>u&08N zsRM<+jPTeY?5WLT=HtB*NtneA-VMeDy;!r!CLd4|A9CAXvx4f|r)=byLrHc0cm$mHR14wpbq#2889FFY zcPHXdqW7||gfS`=D9Lpcgo<~|6!0_Ga1MoZi(Ax?yRZmg_A*7$oPTh%LgvL_J9fV% zPs2A6+ThHC*W4+n=;S|T1QSU(RR7I8c5#Q9*o0yjJt5j_rh;OQA8`c|C4*&}1 zZN72n48jeEpbN!FxcsFL@7TKaHvh4VCgMTOG+7OurFcDJFmL)uNQT{p3GbcghS2}f zJM~+)B&Qh153!Yd13dex&(xZ*x}&~dwed(MYo<1UP;qKloX1M{H(D!oI)7=Nl~*o_ z%0B5j^1ZP5ruTXMv9LiOQ8?@Pw3A9_aE?4}14fU~GLCCr^F>WMqjNPpwL^$53mxU{ zP54X6tJQli>GmT|-w1>#ez_Flf(RFglyB9C&?tf}n3#PbH$~Mox@||SIkW6EfXZ1d z>j)Sy0GmPlAuD^W4#z5vsWtg=HV%YN`%f<&>Y%y&@~wP~4~vufld|=T3>WQ&lu!L- z*fFqQbhPlCFKbl|G#o`KG^i zaen@kH=1iE?sY_GX|5Aob5FCIKNUei-`Y_p4fzL7eD>UDSO29Ih+aEUA3Clv?ClMm z){K0B8|j>h09ttqoi~Ix$J(KwkMV`lrRBK$OS7IRUEl_5%dX3o1bE%lWsQ{@nQx_U z+^8~2|Hf&U3c4?hGsmLjkr3>_r6eJy-yO1Wc8hP%T~04;4M!_b8L=!$>d}AmLStXy zn&!Mct#*QFU^53Nn8^EZF*}p_ySmhpY-$p1;Rk6T@gdXryP@gc)c;wDum|d@L=L`Z z_b%pp+*IfMHUpBzm%NI5cEf9zz5^XR5w@&nOUwCx|FpVig}347D(#e3M@0Wx2pOqI z7h?RSIU<|TOP#`I+f9JdgJEEPfoRb;{8@*Y) zbiIF7o`6>w*z$kZ!TVWK>JUyWV8(Q#xGoaJyh&4!Kj&Lm)z?(mWD$Znhj~RIPd@d< zZ=$+S78&t8A@aU31GddHGzbLb{IMLcU?fdw3gS_gzq^6YjO|0)dN>+8)3jsG05uzW zQ<9wqVrn**@RjrixcT-N;92JSVo2AeQ8#2^$gGY+j)O}UOa$mU;O_9K#rG!b|Ff(4 z>>ldTADoFia3)qD*be`o1>EV1DuouxC_oJU-~;}kxgg?|EVL1Pw6}#@gJ%<_0Kqvr z{)d}vJz^qaGTw#ziT*zUif2M^`Zu5~>Mb^E0 z`{P9wb8@e)OXpHLg=$C`To&9mDqty=_-Odg{)!S%?D9}EyGq3wk`r**!5y#R(zIdM zAJPWp6`vX4HPdftN0cnFN%hQ7s6)SCLH6x&I4@PJ3}moi3_gn-FmjNpF6WoJUwq~y z5(Z;Xvjiu^9Dh-vlnRB9HqNQOGF@z7q6^rO6C$^4-1O}*t(%lgg#Mz@+c-FEo-KKS zyXkihP4H{pE6wZ^ESCIAvy_AJ9X0ykl3r=W(<-?UVZs#Lv^fX?J{Zphfu)H8wE-l` zO%j1wFC5g*$t0Zqv(x_~ydoa`|JlJs@jBEn7haPFdI>k$7ebPZXkDL9RN>=i6~z`( zMH*WMqW^#?d}eUF&45<{s{6sKoR*9>?x&UvQ&C}j+S-53Nda;5TlXPB{o|`L@Lg<} zSU@8vUS+=6x#Msxso+WRjo(2D-PNB9%?AVn=n^M_xW055&G;<1{(}Bo@ZNKEX6k8#kw|A3pn(b?LW?hmbt zXa2_diOCEsFxi&IABa#m^Ox}v5Ys=*0DSA}U{9Oh$ff}zU3B}8LBWR=BfV7yynn92 zr!$V+=I;3zG35>8OQGXvng5y5lM3yqF|d=q8wXc?OhBmgkP27_YBSz8Xkpccr3C5E z!WuBpxSu|afr#Ri6YP;E7q?N55R4-|ds{T9*U=?O*o#mS$ik+B`}1UOpiAs-{yb!4 z(7Kd+{aL2;$Cr6t*3ZK_3BLhWwFDp7k&K0tJLZxxNV8?H(gB%I8Wh#bQPfO$UR!*U z5tAFIrRhbEq=g`dk}v(12ANlp9ze@M&H55C#VUO9Z0HSsk{&1^lWG0TBDg?;H|Z>d zl$rbKEZAb!0z&2!b+{bg>b&Y=b&wsoSWqQ7QJQtM#cxjYp(yuJ<0UwwPHHaqhVyejCh% z5ne9oj~D(Pf2|qu4&3({2t-1$C~kvq04M~uM@wCsFv>&Yy%gkRbbUTk9k5q-MV04r zHWH1U@8oHNKz5KQD)I}XwrTJlocv4(jk$TWby7Dh5_8~7W_RggD^GEsn()Z4B-q|-WLh@j)n#;7vyz?Jx zmqKDG5c!`ls-qw+lO`H6p9M>AU&VH0hXrGszx&JV>CaE>Lv-fNtcMxJ|M1sI^x17o zuVTfr#(d0W{sw@FYF`?hyv%1NAZy)UYh1$drAd~F`czZ?E+o$aKmh>mR?7TMm;m71 zPYfWyLzWPMe>sd*kNx1FKNZBk%dP%=hW{psT6tH>md^3zWh;3H9fY1Q3K z)jH`~2Ro=#R#qO!2oVuzv%^Z0EoYVoXq6o_BQuW(WTwnKP@Yf`P%#k@E)n74%4f7c zQqBCa_N!QhD-=GmRPzV6W&9gI#aYo88X4jBu_zcni!h%zla?8Hw&B`T);ytSe`3fw zlwp}EuseKp8gka(-0WTN{5W_wxo-_BGNXKkpnsd@LSU_I9r6Gw+#tm%x!a9!b6Ph1G+`GhYe1tPs>Bj!#@RbBWCw z{>6ap*)@VBB(`BKRW~15x_QZXFA5G?JLtB$R2p>K_KEI9KigwSJydf$MF{_kthlPfl~lFBdF8sdHx)2_KaFNx zRS%h*q+`MPf>mQikgm5f1Gk|%m1mKN0wIV02O9Q&d@O_nj>5Ws-^YrN%YJDgfSm<# zgV*HjqiL^?WzwpZpC~?gg^3s7d|7e5@|qv|H4lgV)Qa7H$8t@WlCNJIv#_5ZEk8*g zEgb$SZo#?)+;u4zSEwzKh&Ya4E?!{%U8<8NK>bIcu&T6lg`fA%tmF7DeGazMR4vgw zND0rCaSB->^ujXp%Avv$Ki;6)MdgKZFEj;kxeHoY6t0U;vIR2|DL|3UzZI%WoY`^Z zC%tVUm&=$xMLAh()$TPs(qi*7J3C8UIq6E7b7XwlH&`#&dN>h+ws*~FZ&C6+b7X`l zd`QY>zO5Q$uk-w@d$2s?XW+kF`it`V%if1v8=b`7aB1SJWa>Q!WjZFjR>N|l+54wy z-D}Q_<*)p6g{U%65ia+oZO#n5{SHH8Opxtl6Bw%bBS!00WuoZA$@16<1vY5(FN$BA zt>->(sgMjKQ`177o3ncY8+q}YJ~glY3Mo%geE{WeBVIo1W0#9x|0$}sS)tM7x@5%x znEPJ%s|=HPGoz)}x}a!Zl;-;?^fu?fwZ~oMo{N$CC$YOT0?Q{N54htB%PA1YqeoUI zbAvAE27Ls8TKXEhf`#TMKKyXbOspr zjYK9&js@`&a-LR`w{|-iX}=y(GEXkNgpCg3^vMrpbVbF+%`{?^g0F?1U_+Kq;^jan zH-T9?=iBm7Im+rE57OdAJX@Ws02v2y0OTcsjG+;D^9Ljp6+M3jKR|?h zUnbO^7w)JlAEK-_w7Obz%#qOM3Col^2uy(Z2q8H4GKhXKJOVr%izBLk=K;I@mK^F_ z{}JSu~kkYkCV zKf$K$Iptmn)0sWNlb=OiE!lg;>!*JGS0SHv%^68719?sq?C^TRf!Cu3cZ2d@r&BoJ z(zxDdXDjPlD*E>c-@7rP+aKybtiPKqEo(Z1I?Jmjc5~nR1NDc(2&WKE&A0a!y}^(z zZ-qAD2uShgFag?xq+w=@AK004hhT8DTrkE*Wx7=5ih{qi?bTP+heh|7hdkncZp}mg zs-DklS`s8*9k&0DF;6(yqFVm?id;0t*}eP7KUw4-CCE2~)qg*W{3qD>{}XKdpSToz zIu$V%`765G*yF|LXiN=G=$nv zKLg1;zdJI5%|NKQi0QWUgqo4tG!mJ>Vxltd`uqELObTFGLlkUB+`q>tbGHSv6c18b z3fWOmDC7BxaKL3F#1%skWV;hDK^YA{9(o2V`>O>NyKPY-D03&wwq{O-$71Hxmoq0< zqOx=lirfVzl4y(75hUOwk4nD!4z%Le!(6F0%Bh7Pf5@~fV-IbD?#pXsHiG5MA-|*0 zaZc*B396{OfaJWU^2b-@bXPtcO`Ck7e0CF5t?KI@&8yg>u;H29apls~0hbkWHYWT9 zwCrWWCyFO1^f46f+=kpQXzwon(9|j|0qz9q=Z{2#e>SCW3Tls*WRMH<`E#rP0x=J! z4*vyW{!3u@rV54Ip3foNVVS&d3h*efeM87Xnm9?7t}UCV;627y2DazY09_Bm2dh|c z5ONKg`Y`6S7ru0e)A^wdCa?@DSP*#7w_HC$gm}Z&Pnfv;>PY4OCY2cXv0(sasTd^v zS|Ad*l7@t*Ku3>)opMLSk*Sb#ZmTf#l1<-?L=$}+Kgfx{-7PNtWTmt4pO4V}#$W8x zuvk;fE(h5RceB!thkxJk`N--8j}k8408(0Uo2WMx^gpm+seXGVyc`_96_H_XwUfDH z)>WAuPMIr<%(NCZ474IIU!Xr#|KvaaRzYs_X8t;Qw`9}SVs8exKK2jjSU(WcN8=~b9TfevWI{&4{hATdij5AS1EcjliVv@%4HXUz46ho%H z{;+FqI9Yx7K>o=79%W9yoZ7uWN?yGtc6e**IHfvU8Da`ww?B}L<=s;Bt&(wsFIV&o zZZ)qkTU$>I-3r?AQKGG zJ`ktB?{*G6joL)VRnZ4Wpki-P`IpSQ=M0_@EpeMh`rTl>QUE#gsUDF27_{P$309Od zie#{{;Ga7Q?X9#!im5Tz#tNJ#NK`-=Cw#Eh{iLmEHz17nw6U)ZBS^q2e`F|f+br&C z4EvK2Kzg_-z#t<>LaN>P!WJIki#U3Ifx)SFwt)BTIhv;j~(Xc~es*UPM z)aH{<*>2m6WZI&J6NHhV{gv97pEmE|hN~rW1rSRFn`TW1Ar9heX(Ml1C8z}Hi~S_; zY)?^iuq$pnW3CN7&R3OR_zv|~H2GDDx7}#L>6K=VteVDn#9#E(>6_!GS?)v(!`8B) zR0NJq3b{>t1NraGUwVm%?@&z9orZF>Z!j56c-l$FcVm1wxeQ~y6r#GTCMb^G*aI|8-of_ZU9a}weIA%BIsLIJ=@^Uc<>XUJHVJa65|A-;+I%U zIQ55F=H{?R!b+&UL5I+!*;rIa0~cm*Wl<3%?#8S?6?H~_?5jmagAgi+n|trwzX1Dx z;flQa!z1DM|Cqf0#~1jY8(PU@f7?p@$4Y9e!L{Vlm}Nf^KxF(pTL%wTblK5Hi%&!9 zY|OG3`Q=Uc>;?Cge^lTE78P=WjYfExWwwEiV+Tg!j91!Uha1~$Bo@b)QrZ8$-j@?= z%`CbhT{9Q{DP0`bHiECdI*R$VVkFPAapG@7Jww(+(AqPk8>KP+FiK{4rAK>H0`^Zp z`?{XG7yOXFEz%*n?rDv*_luGEc5HaRNOKQlnoYO9z8n$Yyp6%d{6 z&%}pUI=(ltOn%S%pzrj*Qy<>LV}4JA2x_7gy-*cLA0#8cC+44wzv=#EZ0B__<3fy>ZJotu!4XEAIMG(Sh#TRGnQK@xMrs$&Z*yz_fwx>n)$h~dFG1D& z!6MiwIqbHR4Vd~j%EBAy|Ic~ly9AF!j4&!8=4B}2CSrIZ1b5X%_#1{w&XY}p=+r%f*vg(ys6-whML?6nG6Uyvr*5+Uwk-fOvrCfipac? zK-Y7xe&-&)j9Km3EW1vtb_^b^M?gHcA4cj>3r67ik+!B6q{=rv8m{Gspc(a(J})Ew z?8qiMRkop~^-hBS(OocFV0(>Ce0@9YEY_D@K8L;HH*|iP!^;#vG>}>Ma>p@lt7e33 z|0{p()mp$7NJ%<|s0(NqS4+45-p=_t+zJQ2+L=|H$eYm0k({BuI5SCTIMKGHpB_e#08twsDC-pp!!5BF!h&uclW0qUl(&|!Zl zB>5*YS{B?Y4LtgGj2$%Qf4d6 zZJ~ZVwLMm-Q;;DY^@#^8I@(C4S%L{2qD-|Kk%rRAG!tcJT%Mx(wFrj*RWsm0XUCWoVN+m z+T2mB@U>mTqkO(s6}|t+>GP7is%hr}+p95Iy;FuqwYWjG{%b~FZ;SRvn^uol)i*8X ztLkVk#DzbMjCTJ3Jo4`Z)wif4j&8tLrNA#lqS^d%>(Cd>++}ZQ9UA2#y&>+jc2xhh zx9dd*)8dJLq9px?-x3XOqdoD8SNqz}8QFKQXUV>4>E3{4k6XW`3Xe2~A`Z1?#JZCq zj3o-`0riS+D|h5GV|ph82cY*Mr?l2j`8KBb*S=~M(rY5Sieausoh;64=duVl8EHp9 z4lwDxAnsa{?^Rywof!~!_vLntSLSu;LURB$$aDjyUtJ)8_mF_J_~A|O(l#J&<(_%%TNmm6vtYb_-}F=qX5RM=p}Cfe80tJ`|or8H|?J{ zg*T(o_N(^PTcFo(q3UnkFR$lozkRm;CL{2MI>re?XDK>Nt(Vc<_gN$c%KIaxRNK12 zX3mSFNIurH6*TqSN7m9`28Z>@%hffiuOY~Wf%GE%Y_!DxzSJ{Lv$+cI&N2>tuAv%J zoJL!u4LY$c7erb*Hdx!rQjwNjHo|`*(pYFKHj<39VOBI#Ywl)4#0_iH5rYf0QNKh( zeSJ(fHhZ+4%vzv}JTeL#LI6n=d=wvf`x?XlzD=@74l!MFt;gqnY*4VYTSsU#v#Hub zTdA{(#`qK>y>J+V-&;i!quRo;aaGM{u%`9aQ#PEM3H(y!k7v>K}XgMTg((`pHLQl>c-`k7V4ha+q&rZby0v-XyR$v?2dv#NM>< zZ^A1_>V2;`G2%re^Clzq&yxT9ff)Z6_^s!oY}W_$ml{xB+1K>leUY!5jWzblw?|FS zwFNfF$6=$F-JJk(mv{1OGsmxPCxazFz4q-<*^;kzE)UMi%)A>_1lu0v7ZZsY`c6Op z4aB^;%^$-YZJk7K3RE^<(4QC{#|>n7ot$7Fl=kgw)6pv%+3`W&y!P$Uplj{AN2E<= zE8PX{a{aEN5+GR&j`iOI-F%^T`L`yM&|zlP~fD`?(hxIDB-&gYO z=&5WMoK?I^sIhMhyWVy#+4_+g0N3VLjFk)`v#iI;{90{$=Y!FAY-_!IXE6Sb!K=?J z@TRM4uC;sA`W;ortOt`>yS1$$?c4zL9{qK`=EO4St(6jIN2s;-3RI_ii|Gc|FSA~6 zc~0cRVDt#nyg06RyFtl>>${|sV#NJ8rs=4=`T)P6>?WT;*0A)twE!=3yS##2?; z3VqMEK}Q#c9rJsSn``O4g?lHT(6iWBLK4)cWXV%5=|knr-kF{8BDqHY3%=~5%QVOic` z3fmT@k6VUBW_cQCjQPMTjTO);z8plZP#+6AS(RQJI*`;f({XS9)S+qNZmK(o#B^G8 z(8<#1pm(cBC`b>@Ijo~J>MeK&yyC7`?9~oG-CZwYkHGo6BQZj}i?eSjK$%r6GmR%2 zdUX}xHvxBVt`4Y{A#*t-?e%@LR*cz9AoF$O16qs4d?HH%Zl+RQ!Hxe?E}rUZ|Gi^e z(lTzT9zbM|hh1x{%jQg<2a%}^)Bcal;2jEmx5TaFu6al>KTq)99X zMOjD&Z=Xr)WV!s_L*sxFtf}$~OEf!g4C?pZJqXt5ji*U=XZ+8S^Sh#dE_)h!r?kPu zS{2t&?S}WQ_9UM=n4I__hZmZ}PdD5UeJ%+-&Q*7GPEAfcI%edsnR_hF1Wn)BV}||Z zhQt^XG-SbjFo7&{EvuyQGX}W;O`6Zq-`G1af*Sc|CnECa6{5wh6bjs}k&H!DSAz_>=!&^t;amNv z$-q_DtmVm;=sgw_ebp@Y{&2G*rBnPKY5TI94(6z4LZ?feoD_MCrg+?4N9S5!3HgQ&`OJO%S{4iM0lJA+tqAHa?nvl)&VLH&DA8Fz?^No<9fdWS)SrLRjvDw z`fc+~*MM(!h52n>zD0lX20ck{q0zz_y*2*ocSDcwYypowR4XWZw$Y!a%y|+BJ(w!? z@LeRhqb-Ot5ne3?5GK7w=~o~5tk{rq5Z{#gT&K+RF0=&;Z*e%h8m z{j`h{`<764^v7%+XXEw>h@Czx{fTez!5*-cxd?=oa%FQ@=pBZDQF|D1(;VWaowl2_ zhVXQs%5(=&tPh!a{mUZR;#Ro-4FxL97cz(v#TmvVR^K^TXbooLDRRm)uz%LFGfqNQ z3OiW4yh8xH#_rOXTlw6C@gRFpS$?xmSu44{H z7!7I-hp7dJ>Zy~vsR<*`JbkmN)U!j&Ns%9*tTJG$>Z##!HbVt%F8Ym!>>YRpWQ|;6 zAf;#5G@Il}>>UI+-*m6In;WDIT~L~QdoX&5-esm^_fe4nJ5(|eww7lI9iG-H5SAI@ z?sb``!6{&~FQu4UpewOly3Jvk;FaueOpIhpQWi6$M*eQtN4CR##u}U~&vAxtzAz`M z``qQ-DHo*S=}4d6vyU}P9Gc;W0l$AaP**Mucp~HEI_z-p4`8MZmPIhXEZcNJb^4cg ztRnh0@-F9;OXx4!D>Ift5B#NzE1x+e{Ao?B)NhWgVD{MH**X-j|+1+s3E4Wyc}%O5532Y#?Snu z2K)qm1CsD3f7*Q;bUOp~C$M6v>NLZIGx3Twrf)veCz3FX(*YMR0CVD&Y|9G7>MADx zET4S2zoz=}4`XkoYNv(Y1qZoaT*KGo{FTRv_eM3b(+)CXR}Io%=M$0&Z-OM?7Bj6Y z`V(JbwOw;hZ9;=h3w~s8J_bAQ;zS2(o2o&6K?`!SUvd*7ZAF-`Sq980F-l}p_?~0x zI9gC9ry`ZqZ9JhNK`2W>1la3*dOoAl7wf0zhtvzydUnEkG90} z_!{nNz8}LI7(Tmp_?zq{YFUcZ_2VXz1ePW2iUX+|Kcvor&ZQErQFFf(2P0c4c8iy< zIRLLtmtF1z?5Lz}c3W4t$kp$mWjV0W$|hHLmbt7Ky!2#{ z`z6!0KM6pZ*uez-pt(dBJ2)*5K-GwH%*z>MdrMMVI((Y5QwWWzd_HVR_VOhd@tdG< zVH0W681xdesSFGg4svoT*d~PQ84@+3!N;V!VbSg0!h%bZSOLimsG|&bW9C9{K!4vz zxUlZ859y%8FJkv4vD^Bi8!eD`SWse_1Q_yiiAj?JsO3hz?1C_KUL`UEvQ!>Sk{X6p zODYYbK2b&J114(@Iaexx4sX&^-!#OwPPdulswj|&O3 zObd5MqxFw}DCwVtd7dQ1&RV9vG{aAELqV2Mx5-qv>tz;KlG`vV#yCyhOwNGDp*g_l zyhx7a4gwWp5j?xh3=#1robXQ9q9ccZ1GTt@R*OeWEM)k)bAB~UgDfGgO|&@XAs|Uk zeAw0;dStjHqYSaa?VYfZbIG7!+jhNfp4ftpLAl$SjMxeBdjb;DDjoW=2f9D^t?p`^ zB!C}&FivoH!kB}Qvnkn6fku1`O-tAS#j2-94zK+0zIyb|4@XCpjlO)_VE)Cw|F4;o z_Tt;#GLv}m@BiF(kq@|=o!6SI$dQ7vt$&~ZQ|&R!IYKGoi>r!c@Gh5oV}F_LkNF=!jGqzE-#5G{hz z`wSM+@B@Ob9{_X<1MdeChkns~e>s(9f#q#SJRs`vQzip5&jY%%7Mwc6qCqh(n3^3) zHyB%?p%ukv5Ygf43}U|{ZNG!@k(jsPlu9SHn6y%~s}MT}FUdz9a-TA0xpnxXb*`Y{ z(zbH&nOz{*i4M4L7Z_0FT3?%zYX_8~)W7FxDm{EH{RF5pvx(r^!WG!M#Lswm&|+z& zd5wn01TtJmxrj8O;|#NNYw0Jew*@zp*8fYeyAGIe=65xIr3L)0R7iswX|hcC4Hi(Y z4lEBd;cD2>0bY0dJ_h`(jpHU(6BZf(wq2d6%H&*=ir=qz?!~iQ+Uz2cM^}jJi(Pi| zz4XKjc~r-GbIF>0aAn9*D&t4-r9eYaj8HVQ6iVijl0Q>ph1U>h@3e_4JxAdS&Em9J znsbM7$LeMa zoCKQNa$q6Duoa)-i4fXhI}bJ?*ivI7&u||F4P7nSSda|Ps}!ef;mGkU0f{Lz7>B(( zewCKTaoDXnVzdm2zwX28)$NNnx1i6d(ag0X*)|C&a+vT1$MhbH$;iv8mB1V%!2m@Y zlbjsxdpRUT?Wd-2ZV1;}ma`SLtdgCN@fC|t?i!NSqZ@{AG6=~X{Azl#gNA%Q!aW>B zx8F8=ZPCfi?8)i0T!c%frEHlvH9o{7qyr#BwF}ACFUItV?)wP^k(3aH{+b@&u3G+G zo=#tn@m8%Eb9C+>>a(ic3%+tnxcxVg4qd~DD2FRa_Dd%@>{v$S&us2+K6%(-UX+7# zwjH&D17v|+@l6Q_m3m8B%CyXYJc55QAta_%SImus%!Bhi#$A&*9zs!S*HdK8 z;*9uCyuDA!q?mIkvGRbcm`f-xg=P_Mwv>LK&D^9Za}#@z0|h*hrvvGroL$K)@nhA? zmm52JMFdK%hQ{qGw}Zhnow4D1RflX*Gz(yY^|&JP)CMZ~ zLWVc5rMOAYEN&mfD=a}p2wGtm^8o`!Mgh$Nw(G8LQv@KIFd58yrVc=$Kuka1zOc_A zx9qyULdY*x_T$$vIn^59YHnzrnc2v(doj{w%s}X3$U0xi`F`f1_9@cg1~2b#r_DUx z?&91*=$h*no05YFGhz5;Gyu8%m>`T0P$9WCZo}n;3k5Rani@{FSuN*$&Tlr&Devso zJt#KyGYgAM?cac8`E#{f7?-o$1ox>BhZX-Ci|Q$AGe=Y-M9(-2b$)vds_^sXwWqpG z;XPtcEwBqvxhf{6q;rIwc3xC!5Vdf)d`%j1og#ov6F#5oEyxYnKw6VAM~vy?K=aiu z?)?R7!~(%yK*|@&*K!Oq)WYr*4%yxH<19|Hc&AcFrdDdG8?JLE*@-iJ(T0 z4B2PQu~Ee19=Qst-a~S}@As1jysf}5FV*>^#NPQvk=>aX*zzN8;rQSpXxxL{db&;2 z9Xo|vj_N~-TgEStMWW}g>rXZ7A4k$L)tnNuzHem^=q-svl%et>0{|gn2Cvo$+3=WC z0*^fElc^&dxc6&ueOAdr$ayF^vBT&M3Xf~G1ANZB8V-ibi5wI{6Z`@?9oM9Dauo@D zjDkcwMrdtpdP&|rk!aVbfMyK1~$&{ zBLz*QgvsF+<$)^BoOFPJh1aMU1vC}3mF1s6Q(lQtOKW3Mp@Ri&o#Y&30EPY@m@S9k zN)8%SS(Mz<98`*dyI0H=1fCId4dqJAw}5NQ^Gz@kFMEa19O+qp=b0|ah0Rm70ZYb~ zyllh+Vb1Fu*3T)ji>S6kmnEzePrzIqcbYeDu}CE)nKfOTuru`#zOPnuqwEK--#qVr z6ZDh&F2{Nw{1`Y@yusCU*{h6NKbt`X@2YMk1q6_)P1K#U8FDt_3?Hl-4r`~o%Z<|m z#Z8Dj+?@}J3yY^e#$!d`$!DCU-g&!k4btf`y`$Ou9$3LoV5@Z8mkHxm)PF^hB*0B%wjDl4mH7eYw3v@*Tk88! zGlZwl+7vhDQ0&ilOdZ{7mrQPmu&E&dofhh%6zn4GIT|~d&xkx@S1aY+CS?SkE{?Pm z!GuUDCjfxeJYOmCkq&TZl3MjoR`bR`li$WTcY2@=jiKbnB!%?PO!A}-M6{F|*E?8llF z+1#5Z0sdyTmxNA5C4J$0erTap~s&m8Mp@f~4V)B+Wt5K=v zLgpSD{3Vm~4BRa2lY^P?;`J=@hDDG=43AO?z~Bi?P+uw@6nm7N=W#XwY>9RLBM04X z=3uqPB8yTW_9}C@(O5eZc}8iP@6CVocfZ(9`QVdkGeaKVZu2P=!#LI(;v*iJl3CrY4!P>spJL z%;MZR_zAF0Pd(E=CxXhD1%?cqvgNaeuB&GypSe2^%F}|=O5?(?$!VngNCH0ryqw^)c2g(O(0RpFRw0zFWn*t7Q4n`ma|&W8mci1V!P3Wrco#TKfp3``24?GbQ)Aa_kr`wv}dB& zqb_Q&BZRCe%fu`kvpnfVd{g%?0e$_oabGn4HpV_%uLYz5J&{DaTNJpaQ#QB7^!yAeGM9bPRa4y`ZV8mM`K zm+Vg1NzFjL(!D%3J0USv?MN+c%-RiJyc2>!?P2p) zkNeJfG}azX)BnRPee~0>!`epQ{s$0y0fzho3%!uUKO1gT5_=j(J;mtje-PMFLHvgZ z>=kD#7g)Lqrhh86jxT3OS-D6nod>#3K+Q2@SAPOj3~l>CK!>Kkwqyu~Zyaqo{5gE& zAI4l<4^4)-4Gn!c+fvCmAu+e8cgxo$4jQF;@GcDmLy#6OZkt#+sK3~6He3}uP8T$w zcKI3i;~Te?Vh9+^VO8*5O1$<7tx`o6QsXTW(*lMGi|-jWf*k8fH0q1R<=IRlmZ{28 zv$L#^mg}DjclDd};{IP-@Be!&?ObWj&p8vzTFV|vpH$wb_n&Q=_a)EW#V4&RM$uDh zfN?V3_YW6U46g{v)_5LL1j;do$JG!}mOdtq8B2>>#+diSkPK*(H(dgsyOVHg5yC7| zmj&{yx4WqiqCJEhg!Os|Q5{cwo;nSXPuYa&-br_p@=k6gdwslX@>RcQ?~4aoXQ^|;|v(qc7mWk71SfW%97L^=-Ba3 z#%NHinP9eN=(KXru*Y)b`&((SQaSE-EIPT0!@*oIdVYU zVjKDhE!B=sAfJ6bGzqN!su`Lzc4{j_rC7bMQnhhaDRC*6MKoPn=)2I5?7_MzEia=@ z4*`db)y13ROilOhSTMJhxQH)`H(6HPiS&0O5>!ghZ#%2wcFVaBd_0~y8I$6K8ytbv zNN!e-3|KhkDuyX+7;I_c99kQi* zb+EXwl^=yjT@z8014JJ;b;$0S3!{JaDv7+%bQ)1B0GE^}Rd5PQ1-z{3N(|ymTs`O{ zfIgZMQv6IkFrFyE4Ns~p|7~6~zIM0ZejM9J@XWmanm%0`K3c2W^DnD)g+|f;xF=$d zGbVD}z3#(%Pi#B2FQF~hf;l|snIvp%ix4)QrWUtA669pS7cw%}#F?;JVMnAf{v^V% zcLpyqex~Q?z5Ir!QAgpD!wJ;Q0N#7HfOm+*AdxP&#b*H7-J!k6fMShA7F3 zslsn7!2%4h6?M%gJ`tMK{TTm3LT4cfeScOd=u53#6TQnRVA7+!;Lo-p;2XZqpSig8 z%d&Hym$|){OXDp{%76}M5wlF?0vc94Gg#4O{E|hJUYm*JHt`#34%e&O1x58Z46In( zLct!Ve}kZ!)v|_bbh!Zyky`+BiwV=#>d=$pXY8FW=8lgFHvu>!hCoM&73Ft} zHzVVCQ4Mrc*`_N~-whA`bO(Uv3d4@+HHIBXaRui$!$Z_H=_B7)A=b_-O05NB1rKxm{m%DoFe{A{fZ zlhc&Lxee#$Dn_=Fg@gyozNpy@i`V*xT=yv$dloae6Wp`%-kq<#t~|6K_s++;sA-Hn z$%*?nXf|laydVA19sBvm-(5FrV>55(w)WJ^m`fCjBr7LVd{#StH{Y+t&u5F}wshV0 zG53~D1;%c>AL~%Ga6#R~#Gyy59Pszqn;TP_y?Sn+{#Jrv5c05P( zGKQA=G;3a`7}+`M-uC1!5RA*&uXJgiz2K>#&x_R_uS`TnOe7T@8WqloynX*^`l=g`g%Ya!oUzO(eU5h_=pHeF$moaJ)*}W7-_s`PjUh%_5F&pe{3{D!0DJ zB!U9mZ@8XcT+P z{y{8iM_a2#!lUng&p=uR#Jd~yo=6?S(g=W^nN+A5+CUs!u|1D(tdr|09( zd!4JoJL3Pi^R|1|Ura~@W2jrSff{D2FILOdGKarb2~d<8lmoAwRn1*kjlb@0anwB` zaF)6o4U_q%y`<+o(8zC7E!Du#YKx4>cRvLElz{sHRdr|eiRkWjV#QAS&kKVyAxX@) zY%4Rx=x7~P7GF8Bn-?1$TzW|%Xeu_GLZdA`d|bA{8&8!BzskRIFX$;~R4gk3i>av6 zSl~VqF)E>>N35!z6RixP&!q(iaV1l0f8o$nKT{%$Q$7%rs@;ms5dB_9PQRR}+xF7DhOpKRE5hURi22XImjL#BAo@ zo9kxW5YN9_W<=a}iXtbf)=ZTD7?B@*|2vespw$Fuky0o3Ivq^Hte~dq`hnuWoY}IGJkkXt$zRRQXZ& z_YKbOMH`@CDSUyspcJ2066=0x*Tb-Gm(kypNwJ!2X;69Eg2suKxRA2B?hA4zPV75p zIjLAZ6*wiHzVLB4B1VTei*HW^8+NsMVn{_-P*ERuUKcq6g&c34FPP4wx&W#>+%{rJ z)z!zXbKC_J8^^~_9d&hM9-8)AxB4zeplWrT`xlnW9sWCxf*Vy|?^vF;pfw}tTkIgo z6RTAX25lj8f;Byi5`svg{YfG>iFl`uO01iaQaklgy<5LelybMhd%?nw7Eo!yC(_^n z;EXWD&E^SHf$@8vQBJ+^Lq{z;ATM&eqY5wYDTUOExzx-BI!Fgl2R{ob)kfhKheYqD zx1nO0G0`DIG(#$xWvs$O+C!g<;@;Y%@fAfbEJ7K7``Y5%VfJcbh`-`Btr)1wR{42YI94!U6M5KHmsS5A&99@0t z$Nb@Zaj+Fjvt37LT7BK^zW;Kc5!4a)DLU2}9z}}N2DJM>M{I^FinhzS@_)(|7SALj z%q5;Lw*Aq?|M*-8ZT=sMJPzk2f@b40x_9KMifAARU|(Yk3`>9GI_`bb%#Kw@~%@HU?)ivfq(;xed-2j`Np*UqmiMZ}`5>_vh=Pi5|l{tb1UWyoZ4R-YdQyz7zkc=_w zw)qY>=T=2hk5J-q#@K%?%)GaW&nycuuTec!7<}VtSijcJ%W*+FEzwGC30mb0qc?i0 zmeDUVI(%`nP4G=VT?V&!mcYoB7#b;rJA83{n5#mTh;S;qK((t{9R06C!vIzEwZVZz za`c7sSQ(Kp5qC~O1ObDXjh(sVO!D1JDPGKH#v&H*DdHM$XB&cjO2f)n=$Mhg7ZYw5 z1Ufc2cYJ3mlTj*5D z)4Bxx3N?;Hw5)dYI~Ycr#3_rC0|<59A*67;#lHz`x}o4|J{Xn#{n&t;hjv_9oVg&R zzkYdeu4nW!9y$T{C)dg4O#}Te{bTiQ+rlg1$Qtj4qE={Z}d@z9K z&RcS(Xov9t(9Le_=2q9T=E7r6-;sA1?;rf-a#4L9qY$fCh6onlv`ED+SayN#)q)vxqN%q zh_q5xt&o_llb$bN3GV~-y?9H4@VJmiHR|WfHDu;T3Ra8pmZdF~ za{v%(=0t)j3<6~+(>b#MmvCYm+?>iEZ_?tUrG_#-i3MaJC^)m(yoj?F1Q8Bpp;r-1 z4d>%??6v0fp2ZK!S^EtW$&(O)J<5!7g+fFPhi)chG)O{ASS)u_yo!w|Lfoh{@uT^C z&p1X=U;)ESMQ|g0T&VO=FN8q9^PW#KsR;ph3FRB(?v_p`S2-1@XVh>8Zi^EU*qeQU zSaUm!6P-LKRZI)kDX9vYFD%)>x@;T(`koDXSEJ8IRi7kpg0sFO-Stz?=urh^?LNs{ zt2)*Lur(&WKj*evD5(fw$M_ZyZ8t>Qc%U@P5Al@LD+lll{(sPsU!@9n}V_D18;9h(d zuYt9l$>$_(p^(oD+!rWd0*4Pb4Ir9183XAHa7=9nrBjIX71k}N6$codYjMFyhSgxCPdfHxHT}Vt}3Q&dZxBj5X}X z|5!}_*`z+YTdOj_VjQ^;0d$_|>;V$ex7!dOyDB_3MUjG>=vZf37mYvW>087LAR#T$ z9Z~!YunEk&s$COc@biu)-6Cu(d4D3W3XD`Lm5*`=5qdwu@2Wf3TS)cIEhJL5aLbkg zoQ1Kgp(PuTMYsff8iv;|&c6((P42%GQ0*qO*AFd@loOQbjuwCwpT}2Cp@!xQ2fPzm z)B(WF;QS@gC6%@=w{F4iz)GbT{6?9Gc4=+GW1V}q(y z)Cji)^}jw#l1}98tV`0yJrQ$|!sd-xO!p`@m%^kN3sYff&8#9V*2A26GlbKexe*>C z#Xab#V_>|T(IQca&wDq?DurkWon5-q6Mive>cRH@@`I}i7U1)qi}zm4gQI2er7PU3 zix05FVvJb!cfKw+`!*K|?-P-_(4o*36dHys0TMYv#9BD%Mo8FZJaWmzTJ8#^cQrSK zlk**%d%UbJp=?HS_D~p6S%8zqE=X3DQbi(BVaY-tjVhGE2kUUpnu)4&-08LOG_%SL zkrvUqvA(GXb8rd?OCRFyijOD~DoX>2=G%FWFIhd3vCF;VC`$oti6>dz?;a!ZXcyQ} zPbe#o1^AQh$KGrUG(^*uB6ZD*d}>^dPOV_VhFx)pevlglf^zDrjQKWanyv?UN5?M4 zk-i5_JQA4=)c1%A0Fl{%@O!W5sg!k4DrK=w1N`_KH^>sE<@ChfydGJ_j6*0-0V#^^n@mI zn~0R>;%8~*nMr2&g$6A>-HiJvU0bn z6ijv9eXUkvx_ls<1NkX68|&-8uhOuvYH3THnzg3&_)m)(*?~VMB0@% zFrO`0fDHEx({m?l{JiHs5x3l4;&5ahBI=3ioMM{1sKC&3=U>nk5YY^6_lIz&cm52A zde07q042-4&aw9wWXH;@oYy0HpV11!jo>e=)}C_L%Y3`~^^s+ZR*?@_bE85xfuYh4 zcDT}X1pAIUU-Rxfv+jq!$_P{k&Zu=z{kg~=?MDF0sH zcFVFYffj;yJfJDisyN9mO$6Z-(4ByEEWj6)kc-UF=}MQR$%(z#RXwD{GQAiD0Fzv9@1)x%%KIm2W55lr)+EZ^_Y(fn=V>r@!sXau2M3; zi}?_QQ6Y?$ICYZ84POoY!t@6;Qe;i|bUDJ{dxoLERv$>)ag;O5!69UtCas-xxpF#d3fjzKg#gBbITEysUP6 zMkxlC?#%?D9>Vx5?paez(OEM^RhZ`eraQH;0$;C#VO>44B!fTrOWoC>!iKtDLVJ=N zSy|WF^8A3y*5;jJKht8J&Zmkq{4D5Sq(1Qh)#onkn*0tv@?t{)KW2qYFW$K#K$7mw z`W?=B&RLj6*g!+Gfc-FuaIJZlO1GjD@oR%f>77;-^itH!0L^P5YY>0m=d5AKy;4(9 zf6XvC&s+q%JR-6oG3espZ)A;|6An?+lkTqJ5R`is5C)$s4Gu zauQ+%23qh$k8tx2H*(>?rG*^#`{ZEwnzbJLk-BIdZmYLPAGk#jH~=Q8GBlxx!i>G$ zp?wYAR{clV4Zza)#YQk2b5N02*gI|hhUEO^0dB|Oo+KA^weQ?Mv1iz2Rl&l8QT+d} zy>E?b>fF}W8u~ybQ z=X}v8dIYODobN{2oKT^KV@P0`u?^S4op8SEuRP(jL;c8=Oi`&$#J#VAHE7Dh=!jIv5wB&LFv6z4!tPSbSIque$k4q__-WGt0rUXcs>szuIel=0_h`LHAu7p>Qg~#NyI9lDL!X(mBj1z7yYn-tU!5 zWVrDZ^8T45@l$L?)X)&e0ZVHBm3wLbo2cDdD&0eQB?5zUsDq1k8`57VSeH;%jRGeF z)0G#3-6|&sbFr~n&r-wnE`-cD6N_C%EiWVtjt5^T(P8M>s^S+sqwypuj~u9}^&wZ% z0v0~&fQZ~Mj8*3emp$dA*dZJ{1am9vU(r-phuBn%et2TmtcZufrHVDFc2R0xDQ1OT z8v74Yl{oNx3oUp@cyTi(nc3L#@MkePPOj18i+SJBaSVfz{Vd&G+la@FrYa&@B5t!2 z*j`}r=1tQGl)+>QnW9EJ(zfr*RB(jDQ6j&`J6Z|$5wqazW*wY6)<@akCFv@9+W-VS79Mk(QG7Je+! z$tzN-hjA^J*KUaDLq1F*w*Yj#QLjpV(@=O;&@~hsr2*1oCv9Q^bwfVX*pc)z9#pP& zFyDo)+?~t$)^oK)ox{x-sg|}a?8eE zD1VBZOsRH9xKuw-rKhB>GYxv=YJ6u5j`}t$ zqfc#35Zy_jBt_>33{FIfQPSVM$HI8S)QVyV1pPXuhi9i^|EL(x!Udk#BfW9|&BHZ% zz!9DVMS{FWq%9)TvK1mTzi5-6 zZM0D>Td66j0$7z5&*;T}F+T=liDj|7$au>jE{qer85bat24$&I}pAwXj0w|BXPQ7O&-- zDh=Y~wK#YT?&jnvfJ)Fw$Ze8lzPP7u(wLQCqG|i=AZLuyZgxqNc;;wqr7WPJ77mUe zT4%ymz}au2CMG2EcIf2j|`P@AT$}ZVqX54uARGi$NcSxFSDpjOTj& z7{j&y(%Ev+_0*edqTgxA84Za~TD(}1yH;?WU*0F|ev}-cTpSz`zxXC<(K$vI_u!w> z@`LOwkwWddB>Knv#eqFG|5js-v>f;XY5p^JNrnXE^iF`*%F`S~E&L(l)TJJY<5clg z83g^XNj=5Hj=?SrtEIo0K@vzRcSj!3Be4{niD$n^XuZ<-Iuwug7Z3E@HOsUuCLLo% zJ|et8lEO)^WZ}b%a<;0Uo*ES$6(c(`GoaMh2yiHiqyb}(^OwC@_hQ;NdcliIPWvFw z4DSTTpLL)I09*^hReSCcVAH+Ama87X_mrQPEr%p_Jlbz>NE+WcqB!rcmtOF~%D z?nmV(3f+Ih-tOva;{Wd19r?;=<)U_q=w)Xh3B)a#2>xZ=403aExH-Rk7^(s4cBF18 z3_eImS$!?4Yn{wnx_H+;9sJPm$l4>J=JVb@>yf3IbN?7J=3lnYu19VvbznXuH+x~I zPDAjm=gr~Y$9rPXvxNral$C<4^c$c;b{Zl8^SLQhzhQh_obfZ+e=;psSLc3Qvh;g> zqrtjdIQ9!Ym9mc6^KUR3XOHSc8TCAYvIgw0*^`cS$Tyuoy}!`cJwbkgE}0qK^;4AgdR3GnvLA@jXE_gk zRd^U3)=-Rb{0T`R0Q4C~tZ}G!Xo)iwIm+N?A3|Nv|3u@R z*~oT_iSjiKq*$!b5sMHvp??b@-9y%>p^x>t{old`GcGZadn;6N<1Wi zNsmN5=M=J}f{&7ZdhVMJRm9S6_OA|N^DWY_08+wqzDDSz$?zTxAqHDb3zxJlCwQ_4p<^K(o{KiFlAZC1f&7_?1KLNJ1ovf82@plhK~kBeK!RVU z@imQ9=r8kHV|?k|h&SJl7B5ADi#I!+OKet_D6gpG9tgm7%|Pj-RYxf@e-M$A1Yw)t z&cC-p4|$OL0FshXjjq|PNyz^#`5Eix4ktuKjTL|pl~F5x<&rvM9?4_CgukOiu4(O^ zE1%M%5m!57+O-9O(Yk*s$kG}vwaDhhht*IQMN zq)at$^Wf~Srx)m8rmiAzXe0Oj6ML?1B-6m zU45B2&&UA0Z`ct{Nat9b_<%waq_3-N!Dvn(eh@QWi!y$OxG-3=PEqVbpTtP+ zqg8OS1e(F+%=1+;^i{RvkxAqoy_GXk$0MBHMYPhUcgYT>VSf#U z^oHir`qms3xo^wN@X6~bzl^*;UeMQ#OgR6 zeLnD^o&x{4U-_XvtC+Eqr>*)=tGA$n*!SZ6lNRQGLLJ^J%B`n`OvO7QT)bUR<$E{3 z;jZ)M+avw^o;%amw{w?x?_~W+UV`m>>#OZSmv{f`s`71}CGeb1wcJi86*ni;yow0J z-bc+|4DO-k;#-JHtYlbvk&V9JzK0jphdDs9TYJu8I~%VJIs%I{+Sm*CHIbDe zacQPbjy(}UINf(N^$+4}8J7$7hDkbrbJ%0HGef(F)$I~eOM^J|wch11D1#r6Lm#6e z{m+Xps^Hn8suM+Mxo3*;8~MNQobtMVaBjd5ySaB|gbvx7R6z*t!^DrwMf1m}1s7#& zZc9X!JI6gcJ#Pe{0hg%}RqcD)Cy0@HX0^&&v&D*8xtX-B(yrU!m^NZfQdsA;rJLJ{ zM;)1!ldicnG(97HZUwI<-r{%Utr^Lj1TNTzt~oLuVd{9T2PZFBew6Rlun8Wi)Sh^$ z0pihQQZ`ApcH0Jm;R+1i0n>0yT;hP~aPs613l7@3<(D8g_!a>A1jZR@7hxn1Kq*EYvaPrj2})_QB+ydlPpGeMXztdb0*ICl@ZO=Enu>d4 zRpn#!HlE9MH2KX0n4NYV?y)zpUOI~+-Y;EW^o}s&2gJ>5{O+_}O-Hrb_)qNaEY8dY zC;e!DWj|zN8%8G^Fzxb&RTviM!8ivLmSvd&bCUu4A}iy(xKkp~*@`fvHcO^H`|4Cf zYJ-)f0hTEMcwb`}^)n$;q+5pK8>8G}(nTTnb6#=NJP@|pRLK(=2Y2ubQEg5*mpTvJ z3y4Vp?NUNX!7#HTF{kf=JO&}C)}y*xBQ`kBv=|h;f;#t=zsTa@1#;eRTXUZWK;1{; zTA;NP-MNe;x2@c+Z7v(4y1%E&0!|Mbv80lSx4v?PRemAby;&A8H!#Aw43yTyU#U4eiC$4-DLJvx%=iTh?()o zPL{Sw7PBGr^zd=!GtbX@m(59%^;uVI&WR=_+)4YdD_%AFiLY^cgK~OzniUB_hk2`1 z(%z@g6L-B-gX|eS5Uc9$UD?mS$&fbj;tGN`RNtN2q0Hj%(Ygu_%MSz<<_$N+!=8m+ zY5)u2pBd+CJ}ak6ZFKIL>q{tluRD*oeccVsBjQRV`i-l^=&oLRR~_b+EX@ae<3Vl` z*byG;iQmFo**^k}X!a`lc38i)Bx;m4wpR`dB+<8quuCMKU$@k7ua9*@!J>NoQQ^pP z<}5lk{EF?{MP_g?!<~4`&W=CYnG~y>!6K0C&u#ZZ3iPNH_8z%qLujcxwBLPcAKXfy zwYG4fY{G^F&$f6cz^8z`=Qp!NN8Ah@zZ+LvPhpiO1T4qLi9hz11>{f&NqNa|4IsHN zcc(E}f6Rt2wZePpgia+n=|Jj=F~{~+OR;4Y|>_fBbsS|^>I?jVx`k4^yu7) z)kZAW`Rs~m;48x4;)KqfPOmIPV9<8ytrVS&0E{TImItAfFa|DxbN50 zPR>OS&s8%`4}ohmfaO1mS?~Qt5%IRU%N$X4@!d*%-bo<=c6Pu!C*3ojI*!W`8NPh; z#JhZNwGTEr?{j373q|2=3j`lgo@_Tz6N`uaIKp&65q>;Xj z7@Wm+fZ?)&7nt5FXKY3w(-4em-u%;-7Lp0K(=(XSfWX=@ZO0NYL93{{u=UX9`EX$t_z?n;o)07vt^O}> z8srwHt!;@%zRN>LjLJA%Oy2-m=}a#>5)g*4e+MYIu?1OiNmMxr%UN&FeSCQhrdRXm z`dd|Q^J-O|U4LzpV4tol0pg(S=X`@B2@?*fmqbsQ1ND=cL3JhU;Ty(tM4e|Bq^OT} z{s{=`B1?0PBNv8nk#F9@GwT8fW%q#WnMOaJWAk42%a83zOV)zYTKMxa_Ltt)Wbrit z`RfMCmXc7p_**Cb3>ld|4)%IgR~eR{#jDlCLXx9U4QC~UAIqo4Z9h^`v@`0B^yL{N z>1{5`I@;S@pA^3~4g0jG;fY>I_V?p4pEy40@$xPRMPc)PWt)-1to?-WBPa#&RJ?cP z-5u^hjaU!8XQ_ujRHX|j<&S_M;n$U~*$FS@>Hf?;^QHfY*&pX7VX5dQ%JIpzZ(`|o zqw2&?&Q=EW#?i1>R`6RdAm??ue*O!uK!zhaWTxnstn5=JiJ%gOI9t;6+UvA@eEBG_ z4GokQh7X2pdOM+w7g#EqTBnvK!j{Yd*+$7^1jEB|6YlTm=kwFq>PX*XX|RBN)z_ee zr4+D5<3o1rd0d=GyynQgR4%z7XK8~7@YEK3=bF6y-u{D5nROOoHb{FhE)(j=UW(W( zS+sV%lgFMUG;H_i&!4%OS5i3*krjw?9QvT5(yqvGC;WGVL)T98n|;7Hi{vL2b&XYB z9-||PQLm^Yk2Uu!;PZ*)*^qvDXyKFg^f*oJ#Cw-;Mwp=n$FnUat8?e?z)UC7_l|+d#CA!dg%)u|K zoxqv@K_TjPleVb}bPzj1`ZtePU|z;3E}z3OEM!ZBib2$_G;S0p8LQl;?RfRt<}PuQ zx$bzO>uw>GH!;_3u#W72i<`WP z6L)Jpv*^5@U0`ByDhJ971G8j^PR?oBJ|vPd+ddP7bVQqC(s1j*((ZN-d!H*QSRMf& zS+Bf|hPco{V`WXIj%zPp zm5t?q1^Z~shU&%wa&&yjxxbr)2}4DO)8nU;-WV#u+RF`WWR#zGS%|Pmjo{?@~4wNbP18+b5-x@)cI33zj@b+sTyIL@)rE# z@SYD;tEN);e`R5-LKzyF@bNzpw61?$H5~sVCoEkaXlIXc_>02LSzl7|ZpER<(zK2= z)5V6i-`ba=j%a*Ot4Rw6&}Wdp8yUz@TNzB)*A=5MP#AM_M&oR)T+)VYmu2B-pj7ZLKbR)LhXzxoDg1L0?kv_g z0^UtzVXnBMWU7w=r5-$2ahlX4P9D)pmdpqP!xC6JCWMAxX0oP9Bz+u?Y@yftP&5XF&m!&i^bM`PG6{u(i8O~I%9WIF~#$ly`pm#%4g`ujo1uDar=-2@x z-O5;sGGx1J@Oa)g3nEe0pZoz9Xcg}1Oh1|ZRBx3!!UA z2VTo!4puy`_!S&MY!~>2ms=;6SCQL*9~R;=bMsg=q zi@^<83op694%DW6YcKdOJm~uN3qWRI!NGf1z=mkIc5sebaPZj;kg}>c&ahi>uJ zih>ZC5<)`KA`oQ=LIi{`NMgVcNC;CP<8%48{XK7o^}K(*e>`iwcdZNSW}S2HK6^NO zpZ(eArd)7!*s<-KZJRc2+HwAz-Nj9tl%_Uq+G73X7r-yX>pjnbf1gEPbU3}KmVST> zJp3oMa6l$Qoj#&5D#4GQx9oiOtma5?+2jBCfAz(V&Yi9%hx@-& z`tG3W!9%8BZaH@8`Rl{Koc`h9U4N73x@C87U+}!Jw15^x%(t$3hs-N9=`Z_iCGTdI ztl=kvdPC-8@3qniLy28OHNHmCYfo#Shm) zgZoNslmHfNT(|L`F4|!awd@VtkMnNtjy-O7tx&z+lwkOXWo9?JRn%kj%LZ#1e>mQA- z9Pe*>K(}B2&uMMz{aJ?D8Rns5S$v8hiQDN*%{L;g@`6{?!X0P)qM+h;Gqo<&IPp!B zIsKZ*Ns1_`CK!|F-IDi3k=*b{+O8_bp!&km#ncf5UAsSa^z^$C%;Re^VN5BWKqN8qKZCPLO{zW z;1x@@2&iS!%3zJ(6S`llr)ARMAlE|;UQADIw6(1^n|m>|*5CzwLSLVY4dw*G&#OTr zJzAESqM+PXztTH89(%N{|9HpkyHD@v{G>c{dhA=FK4C1!7Rim}j`bz$6-Ns_?QLWG zJ-Ksfl%MI8=IkLwU(1KCn9jS9N>g7DX5<^hkUgZ5k>*$IN?n+Z>Fj_z2O(EmVq}Jh z%HHl}Ymc@k-hndl3U{&F*f?sY7_nMhIoZ5kE@hQmQZgPw4nq-p6@SVs?|QE?Z2jD* zU(lK9oKJmrwCSh!HM`Q`RVhmc7dg1G0x$S_b_~BtpJAZ}&9oit@>V+wkQFLZJX|sk zktw#@ed&t7*e8_T-f8 zh4AL7dyNXYB#361JC-D9?J$2a)*S4ZeCEOM?Eb#^vx!|EB^RB(N{bvE7@w4lPgbZ= z7Vb6$8rU6iZzJ(rcmqNj2s5!%A}TkhuJ?|&&zL0l7MK<5{i!F@IxV}g7*t|rr0+Iu zvM#ojwHK%RSREXeBC`;Xm+EZ_-I7^*MiO#-@S(v9MJH}N$A}^o1Cul^k2Y$tE=EoT zav$nWK;r}5@m8%XifG%UF!4tVj1vf)}Cqh9h-i=xAXm2E~h9eIEt2K3hNa z(8O~!STA* z;`XbqSQFdIG;RPT@-muU?K4@Pac&r43-=qwYWaWoCj}0cQCjT z2vWCo=?tVofUw2voOFXApUIpOP|V>9Kz(gv24m*;{?1YZ$S&KpE0(MWTB(bpRi4$U zpEFxxd8rORXrk7p(k|Zg{j#p6t~c{NCejDcHH?Ia(GOo$NwK1FD!r>xeuTz^l*X&s zfGt<>?xS>NZ>j6%HBu1wIUqQ0Hf|kL6Kw2)IVpM^!|Lj**yF1J2M2~sEZJgg z>lT+f7F}C`1!Hx#nsM(@VLkKcBky<0u;X6r_Wv@r^nqv)hicNrb2^PXvIyd$-V?OO zz-ok#Q+#ADDSm7&8v+~If2<(SHZ`6zzNq?Gc_Oqi&7DKN5kF%E&MMG&Ui!6H{&Zt( z@ouw6oUe3$uCb~)2G#$`(YvFuH*n*@U?|ClNA2!j&R(ztoXH!Zta{+S4^Gf?@Xz({ z@^__fUu%;Xu2wZ`Cv_adgy!SAJVQ?obip6gh5RNc^40*b`WwXyzZ`zu2>2b#W0~%P zaPwLnC!?5fGG~=#6t9Dm`iVbwISrhjX^P?pF?8tO*}%L9miuTWMW|UUx>9#sCP6D+ z0Zd!e;?hb069korK?81zcC@pCk5(wksR=XS@&+S{W$~?{+2xcWvQF_*{*V;og0U7o zUVal@+9O(znXam4xWF;#R-4lw+v`tdZy^V_#(qp*`gQXzXP4c^luu>B)?N6{3@Wrj;iolL~pvSR^ghmS^_tn%Yu_bbDBY@zf(S#r6En3wl-1U#qjmN;g6Dz3yHo zR(ZUh)yrH_pKwbe_71>9JrK}38@569UfMH4-pOzeNE=UYtVwt3_vP)^$FXfnIx72* zB<^qzO`5BtS)R$s0T@*Qm^R{7VT=-vydAclkZkROc$J27UcMH^j!LnbFx;j=+G%Ai{FdU?wx>Z_&B6+ zqkdQ%YN7mmBJ_RUcusJ4Y1RTO?OKG5f|8?F`aLhhZEqt)Td$ zIeLMCf}7^ATXdx>6C9%)4^pg{qj`4W?y={}u(1_BQji1(ejc1+#F&$VNzs$KL>@nBAXv@sv8!c} zXJ!U^wk?K*HF5-jC8aTK`D*2+MwCPIqOCYLsxTT{;wphFOp!A`YB6fCpd30{pgwRu zLI&bKQmjA*+~ltn`6ZGN>W0@e<+6jIdA5;p`avDXj9hh8L4?KUgZ6rYcW&^uG7i8%4s)@xs!q{cH zv!2nmgaE@>y{SiMm3IBkvdYd}s+5z-{}Gbyu-(+~-(~yjjnWH*n}by>w(bl2^M<)m1+l!Ly@=extjnT)2M{#WP37}4 ztHlpL?TYJM1pxn&CSs9xBa?8wt$an-SRS;(G)LNP532 zjHm-h9%Dxb#K#=yl4)e+ok;J$$er~|?l9x-(TD6syhvk>MFtgNyker}g7q0XeG+HO zj{@7DcP7aquwmv!k2Ra8=VC;}`&cX21h{xPNpT#x;07yzt@8R@`(za}rZZH`m zx#BM>wF-FE=@HL!7q0V<4I2;!NsyYWgP%c7D{A6?_zs?0p!b-Qda3wYF8Ing&$T+M z(cdaD4LKi+eAZOrH&t`sMDy1}&{&KtM6GZ3Rd;~*_?&;s`6z{%T6v+N z|GOGosr9P$^FD0nZs>TEn1#p(0!yu69c})o+Lo42-}iWK$h=o$Q?zQ?9W~ig-6_Lt zt5(BnrRvHW57yAJ@l^FI_9+YGB|2|EOjTI$N+;9QSnMaE#|9O_v!>H+0s?&^ee|)I z!gQOJqL{W3wQ{t@Y;z-P9_OZR1U9Nk7GWwpCZ$PSQ#2KAxrCNQiti6o4$3;5;6KBp z)qTza#eET$O{W}1E!IV_=%Lj_WCIHbU|d!0?VYqeRTiztRJBUIyr<3iH`C1Q#G)XkPwwo zOGH)3^6`daV6%%ewnZgpeQWoJRKn zO-pg!-VSysIBwJ;xPv=W!FG+vT`!GCB+t;}X@tc$(MDb2-G`NNXUY@qztYOEuD3F> ztxeariVD!5>L`B8O8b=1mjn|=h8MJ&2_NZqAu`uan)j9~>)1E+TI6OChI$?qhID%W zzHdwdHjVTk*`teXNg~oD6tL*yPVqlE1`h@&btTD;YwD}#>y7V-KM;1N$q({^l!zr) zmD*iaUG5@@Iop%2epH6JPv478l2nvoi_0^`sE$M&R4scS0gtvW59JU^_Y*e*lgR)S9XGfno~X#dm}FTTd_=9>I^s5BwJ0a zudFaiWvw*&7(C6rEzPdZDE!yGh$MBd^0d>wP1Fm$7m2=pW|1_~bZu(Y@lwhK5ev_?Ulfv2Ar$aBjWPeIai zVkTOCD-ptDQx&gi1DQ0;1!YPXx+-nbVcQHV?}8Ll+GpoGysQ;JSm4*S8*b zc&U*qKl)B0U+2)i(I!vN5HDW7et8n9#5)E&{kH#BkE3k$s@L7dF?r0pT!t;EfN5GN z>*0I!z3fbY7?d@o+dj`;5QqJdOF63Jo|@wCv%Kw#6c+O^$eyXf)UYS(Do@*a9~Fsc z$4nH-23|@yXO*GSbHm9+Gskr_jhO~pU89u*@C*y1)|FP!xrlX=2uqC@fy=?`Oi_h7 zh20vgpa#nn5%St>Q8`##O?4SAYB&_uIvc>;$ zHpJ_In>^YD<==DJ!}c$@`;J^T{>w$5*wG&lWg&0WII5L*%x{h7E5jfrwG`=;Hu?Jg zu(M4I?+6Q?`d3!;mj{RY5#GI}?{Bj%C;1T{CVq>nLI+p3x-^9uu2!`6%q8c~a{=2K ztxIu>rdhxft7i%66q-rUuXQhs!Bm@uk-pWI>)vhZFOi;`37da6o4kg_-8iBZ%l=Ib zxp0h&#Z2sPIo=*eEAuM!FD&~7VmRKh15;}E{=f-H8ni8~y_c-Yn!ezq9hpo0WN|6Y zH&Dsg^00AYe_Bh5L($o*I7VBsFayB1=9kkQ=m$1i)i;HWeknR3Z+jgjuRIY+)5uK+%h%{Z z39lUE>mI&7eKTjF0MmD4OgrMjNR75{8!zj_v~QbwBY!SYzhDam5Rq%$b8JsG$%t!#!^o4sHZ`~RikuP47W2aA?Ko!px!CNjfPqtua=ntfp!SZe6-c=8{JIv>%ufMJyI1iGP#f&Q9+%VSi9oz9A=H+S+v5XSX||(yVr1E2 zN?rRgAoh3?nY+;CFmTqp9Sx7daYU#;-PlP}b6F zR5g3_E@_M2R5nKR!m5EjTAv6a6zXfNsxlJOt`HY}-0@X8l%c>>Z<|0|@|zdf?}tp` zm}8e6dM1lc;LF@hs#{Bj#35{EH@%l(l+~0|Y)Igw`OmX2f_EY_;NkZzB{0yi4`gP^ zAK1*yTip98i}#vRIe1l;Apt_n+G_m6lB=z>`sA8fUU)GW+dG#;4frD|Yw+-sYI`xK z^o43lMzuC!D=7Ka8x^6yFFWEPktDLuJlffT&ygs5)%bWO`vHK3L6DIu=}s;w#LpB^ zk55G(mEBl9y{6oDR0lrWzUBRp?t3$=ep-C*6kwQ$nPRvzO#@*4I}>9U2asf7xj&=T zKtocOi_HLu>X+Z_0R)Ybb~9*RWCIO_5cv6kgcY-rzqL|Ov{Di?l_U=m7f!BomRWFV z-EzSUSRXl8CaLh~OZK$neuR|orBMx#b8C}sG|byB8axMZk{oWi8@>~{xR?9TRN){7 z61heH&2Z!_9NI`?M6%`#?aB4G)f9b?;HfEAtsV2mW%fntU|r!3E!gDcv*FU@V|-=U z16Ap__YVvZZVbCQgEr)A2syod{;{t%l5aFz0tQ2o`|HxLbHn-C$?sKI3IC@$(VqL_%wX67H?u7hV@#N5}84VxjY~^vXMGI7qZj@46&+p-opZ0)qtX z*UDPh#`^~65mrJMXD|G2v{y=V9CsUcA%x}~MT>$*?k(LZw6{5okB-S@m*$R52-|UenD=Kd@g%kW05+*~^`K0?)LKoz-U$ zvDB(;NM&}75MvoajTy0saQ4MnnLrai7BWC`F$4HB-G>FECu|L_17TR z-!2e0xU>8nEAoXf(wmZ2XxH(QbD>S&v{#$_y1)!uxmQOTZn^q?(na4(3bW)lPde%Q ztbeLAcBLIyVhvfW0SU<`@B;ZdNbW1{N)R}chDu8Clkqn#B{n!Gw7gW21yd-J;>2l_ zvPa8RGhlPs>kKJrxhq$GF;M}g=oTjrI5D1MTdg$+ECfTxRVs@S?>BZDrj>=70y*OR zb4td__BD&!Lx|I>%m_@^=dk>>Q+p=oN&p`0B?R@l>`96EqA$MFG&MM&WbF9%%#QD- zIxOinqA^x~eH~bs;QCdD&;;td3)(UMR<-iz38peElF-75$C~|__Pz2$wmlhzC{!n0 z76fC9y{wHy546&C@fFE2;mgOP0!i1EaLLZ|eg<($f|>|qi17W;r?0YYo^awfDs%u6M(bdP2}l;f)pAgXV%|9@lEj~Aup-;2cY^W^ z%vusRFQfHv>-4UYT-btspR3yh#WIXK4}`X{Iu+IF(xqiNI38sw$HAZ2E<(^hqqzIQ z#qtca1YHRdKLwo!uj~P5%6Sy(&l5s(MN`up1ix4xGs}&*QH%z^4u5t0x<3G%rrV_) zrC8XpPbl9dH?yIKDOa;z1r$W8?NqD214)fY7NA6=w8zjfE+0S}-7xDtL8-qZI2nmF z>%27Y?1iWd_u_4>8TB=l&1JtRYyPfQN561yFXXZPX7u@Y)X;k#Ekd*SrE2b}8Em73 zL27X=thFT%Z)t!>*B*(~A%DuVTQ^D#;;?;leY>of_P8VbRdZe8ovyqSh#B(vYu!Vu zJTx;5QW;>oO5(mQLs@Ca`V}SVmSO@!Nke%r1_c``SpJQAd6gZhj z;-`WvN>Xr(;&B;6gtO5OYh5pjc@2_=i!(u}o8sq~WFPSeM$TP+h*gA&-+<6Qz;E$B ziR(@Z56i_@)F*s0Z+r1dx}eCn2LO1_0pLhuMp>vDq4OaCG=?X?uxq)P#GMi!z?{8q)*Dn64|C?@~e^6^T zrj1MfY92_i6$5#fd`%#WPgjXHPf;#$cEE>@j9G$8oU6eSH-JbKSoY?#J~!&}iy-Wb zlx18AM-3SR*qR%)0L4*5nb`1VN-|4U0~98*9(4T8#347iUE*klt(k01Ap%!oBqJEZ zE(CZTgg#_r;~4Lz7-1>EB2zavLA>PK4og;YNG+nE3;?E|tepk9xBU@6Z_2{}b~qRK z(Neu$R59M4a0oe7z-FM=`})pP7hk{~`{Z|nDiq6eGv)C2Gx^+LupGoNmP&285R`C) ztA<(MUSq8_UCOFe+UNb2tf=8?Rn0L>|BT zr*qk~X}T6j#tMEU9O!mGrbrf(v$-Oa1To+yYj>-}g6D%f0@bU5S7uCXTw<8p>iq)+mZ~bD$u=Qin9hQ3kBF0NDHY zKO6^YG-e$BocE@(RJwhPF;FFOp^MgOyZlzX{DTO6P&PBU_GsCuPu#KG-Iw5+NH>=H zC(=zZo>>bp@xY`TI__6YMUbM+xEzBVbyGuhOPFUS^IgI0o<@{RJX)U<@ZY)%Adz(9 zneeBB-;9tD0>1PQNMh5bD|w%M>}C$NsP+G*>l9C{y?3rii$mj4MXk&0ebC3QR`ZR^ z)8?&uGCEpPvz(esIgFGqVa^Z35uQNWUZY4ZZ$;-Ck(LLBEX+Jws*M6H&H{BL-wpm3 z*VnL$i$U4_zLIR(bksSBtFkBi?RWcYX^ov-VmD%2{GsjYfP`m{1IxmGT>j$ylm~yk zUPrmrTJ;-D>U~Ht@B?Hs>mi%^`nl@j-v=v)QPMtI?mKjh08FpmacS$lzmD_ZLE{si z&x6;}r7YiwR3!PP|7@JOp#lGF{eR=o*Blqy>-wD3!!1p1#BXKXOb5+Oz4K(4u%&~Sn*!B8&(w!3KIWbyqlA{zFnIoi z?$kd->U@f2bd!JEd^}~?7)Bf8=`#RT`?~}(^!VDXf=zjXv!A~BZ0D>3g3)c<1^Ry zYLnyQ#!-}YXvg%jr)uO z3X~;B!z8M>{*p(#RCH*yBwUZH*59T1r0>Vvg{m!N-BC{E^COjd9b{tgwHp9@A^Hgc z)<(*3C66ibA3O%MBSLj-(}A3}<^1SRvlwHqIJBsM{^q#WJx(R`;hdl8(#p#jVNxn- zG0cWP+`?u94vp<-g|lke2haw~YB}2^kq`oNORULjk!Dj`MXS5wI~!5|5UfimbTMn{JTNR?`Za zz?oYBj&K73`;T3I-f-9aC+Nc!5hd9J^xonvbLWGX#92e+GQ?4Fs#UBps+4IIQ`dbgf504hljm?(D1WX8X7WkKoS3{ZaTvYl8N`~HO#DzY8#FYK| zXe7|tqeYuMPV&KZ?lF~UOn!YjWD=cOGEZeI}Ffy87{uI3tGX$ z=3c6=;wG0WMw270x*LNKqewwhu;>*c<~sd-lg)gySk__91S`@TXHtgDdHjS{i+Sm& zcSmBX5HmqV?!44gIf9&A31$Wxb8D@-@*~6B>$lhSW}r;Nz=}s5YnR~F?^PHRh<*{&L*bKwP17j}Nu30W zd^J=I8%-E@A0IyT7t^~opr^2%l=^Z@YSaf-2P3#v(AfP}5!T=CMGwm-M&~BaY?bXu znt@npKeJ^;;0CFa{IvmU;}`rJNOElulmsFZ->1JXSrm*|)X`^?k0}gp0NWjMDAnpi zb`-nIiWMdfMzyFFOt&I7z;P80OYQM?)NaDVPC5p9@$1 z$vo60Si;sra(@vw4wjC$VtO-@r0J`1$%2B_s6!#(%b2*G`-7BlfWGOzM=FX<*(`Q_ zj2dyy?YeEPQNYtZfi3H)fgf{PzfPfbvLUh@54A;yAp(r(1VgCLRG4hoDAX zu!}U@7AEZO;%^QA?p}wnamNXSV|dbuF-R!$boN3Ev%iZk@dZEg*Qg0-khN`gm>7ai zQcBH25u;1*$LE9jt+gj@lTJ)v@WsLbf26%hZDLQ?3yaF85^yj`K|cXOr-X!yehnOt z@fS7?9-#7Sol-d+#y?r%War}Y|GNI974s*n)Y1AoNW*BuFpKGt5pLId8r2YuhMFLS zh>0cGq=wo^M{0QZyHm^*55CcDahy(n)r3-45BePPCVsgc{-uqtA%NdzT2 znm3V@IxkcT{Y(s~OMYfuG&ft{^|0Z11L68_j`0wLqq}>1_oIxgDE6Vil!z?SR4z|^IbFCY ztRJ-cU_W9+7&m$0)2=UgK=|kw@0*^S7&3O=KG={_M^mtu_Gz*SB@2c2N;gDeoSJqK zFu&@xX;AM%b2TIjTd-qp?c3e&;$u_^82l+@4Nq+h3SnozGh7OT6{ zVmn&`)Z_ftMIDx9_7qt|gg##DIQnbVr4Hy&+p135qL+nrjt0|@f98c1{~dn*<78DY zh&w_VvhI~>G$O}iJ6=X9YUSV8wSMd9JhAfB6D%9pIM614t zC>{Qs7;u4HTeEb6J~_IO=y6-6rIJRSl0X{GI(7+Ac&~AtJ^jH@{kqO0xc3W zxmjniA_h5=LES0{Y>0IJcxs|Oge4THBGaPc6Ch+1`X?g=%2#T0LLQD}D~$*HTLoNR z7rjTJ`*MWW016SYi3T$MkLHeDz)k>wysPuwlyF7n{F~;_?m*I$KDCG!`lzMQo}|Se zvTT>ER;~)4F-d(Py#%ca96J7b35u6+m_o-_#4xU|Of~eJbc_ebi=3Z^U--=UVyikV ze=KHurUpb7AX(vQZM)UaPRR;vM)`XaV|#QsP_h5`;FSCIox*+Ny5sgN_SftxMkm>+ z>JFsw8Ax{BgmQ%cHC`TK&&5iwdVDn@y%FMu*b72?_qijCpZq!Z^3| zp{o5e&6-#m^i5Z(uW<8`_QqnQ9{NGYuBUIMr!E)c5_QP$G4FAg9Xy;NhP)kPlnA)V zz-kBkholS$?|mccOyNa8B&J}WI%hJQsw z>wT>U8>X8W+>b?b6BZ@GOmEyXHxS|;&l#hi*Ae1r&BuF$>TfR)DrTd)(09My(zY0a zUH_(cZWsV3$Kfx0X4cLzeO%EzViTvmIj^#aaB`PA?S-@GeJq|hiO=5O_yWa7_$~Ey z9Q|9g=KUR4L>OQR1|u7C*Cb0=*tjX61=3=CvwM$=4nAUnK~!?7oAZDdZTPUvwdNvh z^ZDL_fB`!000ytA;Ehj}q$ zVlGIozXH@AQ->yAFQE;=sH}GKRKM>cpdC(1f*vfkaav=Z#yESPpM&1-*QP(*e?YB+ zr2CUC@|wl&;i7-!7!wCJM3`yE!8SSYg)z`mBQ)`?Wnv1$wKm5 z@%z*!e`WI#!d{!ys1ED?7vb|U>#r)mwr?R(FIYR8qHbha!R5c`Mf~s$i;svz5MT4} zQ{2DO2idfQad<6^4k$Y=dFmxlhiP4Fx-H6mj&t9JsM(Xmd zMMSC*o_d*%puYF%r2Bij$9*Rv>UPBDfDp~Rkqmm-xo%&w>MKo&EANh&v8wSIH99s) zJ`~r!i*kwyX*9Lcro@lksLRNex{$8y?Y`k{1pYbz!_NKjq+Ga;kA^XA2|kkj2%J7P z&FX!J@fY(<0p9?Y5#?vr!5h_0h(+(eDIM6O2mk3Ljl{4K_pYwLO?<>QA=Z)O4M`au z^UiRntJyfNxR^QG%{!bLQ3of^KdT%|f1kT%%{L&*mIS)=yNkJH>!Bft33{9gqlt1| zB>lkbDBj=QZoH2(e9Iy%npqHRTpgRp#IdYwAA4`%1rL%!|A2xZ=7mG{Oi_wie#}*e%`giylZz zKWBzOFhT+~*Wu?ld8h2jktz#^wgY7`oX%{$-)~RTLS>joOgMktnRJlwwKn-OJ%D#{ zlu{?VJTX0E(x;NMK#9LXayI{D6ffHrpEM$>*LB3Lpz$LOazbhrtCdTO0+atyYo2WH_voSnkf4aFhmTZrSz!i1BApOHrt1|FQ$-?NIs6l=qR4;vRcX}TsL$& z*Bh*5?_G_FzZ5j)j$67sezVSKoByCpxBIHS@a4)+Cs5DqpR}%%mvzqrI9C>}VRCi8 zCBQQ-etq+?=gT(3k@>Ox1-2POoMPMqYiOyWlS|~`3UY~=k{8z-B zVS0%Us5W~vRXI>a>Av*PV1!lb;#4t4k4^<-yA-d&Ska?*tmV}f;fn}T?b`g+`o+5x z{#vCjXFNG-p6B@2aQ5;3Wx_yn{{{E|q!g+`5tqVR@(9wxNGbBTC}AlA&#{zf%cp&% z!A+*<{ISW1G|DL~^(A|B7eY>#j_}t24>A$UygMG86CDM}a__>SBzvWZSLRGmF7bnT zP$lS68)+TGWNVVEY{}~{NHz!? zduCt`El)5fM_S**JL{R(78oj|8~aUy3S$}{^d@T|)eFHZjB_vHj`deLk7*m(5yNQ} z_f^5iPcPj+<=vSA`qBhGzcZ@ls{4si`3NC4?N(qMI3^^ipwslN7AMWesDYH;v#U0I zJ%pAsMB}x9FQ?8wI(<=}Qx?DuhxUJNjK1KR>oIZU#3fN(bCP$61W{|#r2BC%LN7$N zX(vZGM?iA$o)He}DZyfmzF?O+9229P9+|K+%xZ}<5-gGVySvnphZ6SBL|D9-tVvGe znr4(43XC8{Gc}Qs^laRj$nFn0Cx@%LQ=;qMuMUO0lWwVUMy%&A)fG&~Wj!wu^ae*U zeNZ;X39)+C>G(BDN^_f{}s#$sg98k+B>kYi1 zBv=GH2Qzm8xN|qm##G7!1@|_TL4Qf>qcV+hDj!XThe?K+33IEfQRYYjRNP%N;}j8$ zGz`ttTgrYl1K2kJtJ`ann>WLRmpXH<2UYeA=uO2^e%qxy`3Zp%K45Kixq;ej+;!5> z1S-Aq;A)4KxfLRjcQfqFn7wiM)He(j?dlvqrDr5)@kKX;Mm8ueu#5Atb7Q|aZ~TE? z1}4!;8^&fEIlCqrMdT90mCp{uS)jXe#g&0&l^Xo6ml8({>*84yhc5RCm ztgkuxs)Yf+5+e0y5cftbsb#Hg!8SJp1X9LKm$ih(TUoi4&D;GGD8~s6ekguX^$}6p zMN_RXUZ5aUaZ&WiJk3feXNfQF>;e#O`1dLpT0)-1^~}X0eL})}{0<+l@Z<#RLZf9sAfYo&L5^pU#E4aP$kOX9n&zpgS&O2%UR`(LJmjz3-BLDpqAwwue>e!H+U;{+n>ercu#*zY>($~gRZ8~eV) z>+IVt%qCe1%gA>;HU{+>R2_0L1kz{a;Q@Xe{!eky%``tHO;bybBgpakQ zGEMOJ-5}Ic>k=6rH)%UG)Z~mh*lByB?*|lDlxA<%QP)j&6<(gaA)&RfGCL!g}WetsXzNa-a141Fuz5x`LbN1GnBWc1`O+H&ZLBbA!%r8ujO-u#H#cQ2}?*u3Pe_!?q<_sh)b9q?3= zbH@&+6X1(I4QwaH=#u%|F`3+JLJ-j{G#A!~>@L1P+YBTkNFYj?+9Ep_qT(XUo?Zwm|uY@P}kgfs2 zb7cQOxa{b&^y*6KQ~cOx-$FF0SG_==AwJL zqHD*dqgpM5;;fk%kDfhC(?f#>yS#>t|03>VDa#Vy;3Yxx_%Ucm%REN@gk=;X6!m@j z{t{MyTT;fP$MW{lps!!W0U?WI+OA^FYN;4gW0Q){3(o=0KvLSF3st=DJmfZ3*pVHX z>p{T2FSFimSP&i$tO>yJp}L1-Yo}6Zt+4f$EP8SxN_zTmS9~rtZX)B62Bg!yP4XKl zNb|f^*Z6{eL6kVp+`RX6`TD-mO5N=a`;7<4n_-NTnohns{|Oi;3W541HDjRSZ~fH~ z6;7GsQyuZstmpp#b)uZ^sgDU;~R(I6SZb@BjWf`%Iq)xqieqrB&1=L zSKS~O5bg1cgJ%jd@Ys;B0O13sYjE|3Z5yXP=xD6r%(bi(@omp8wVqTp>}Bhf>7>V6 zJ5a4_qg}}I$eCBOHAxOGlkm`Qwlv-CO2&Ks#oO9-=H3;n;|*GYl$|mE3dB+|yvX>d z%*Z+0y1nGxv2(!=<`} zfM}Cv$p^xuZ}s?dgWhf?wnO83&GRGD8<8_Dqn5y-6Y6^iAJnK_ybr|eo#?b&y!xR>ty?5OYs9#arRYZIRwGPJ{ zu)~rC9s?qI-1P3oOy0yhE5*rUpE4WEzM+-f=nl0?s{FZSr5VoK55KuHBiLzUr|%p+ zLK-O185-`o>b_5(KcR}4NcA6opiSO~pKr`9AKx`GX83NePj~uhpOx^Wx;emI9QXMA zt91$QOHH{PFr-R~ zt4ttt0;z1T(@}C zQ>B(r+`-FBLIx)OE9RKMdOx!l;huIV*NWu=VXdSN8~z{1KJ5M>6~G+xdH);I)cauA z_b=Rd@Sw-!U#5EKQ1rupB8Pyn9{3mf4jnoMd{^NgaOVHbL)VvYPBTPkrv7cMdaw-O zsRrC%{kx0r|3f`wi&-JH#3LyHFy9cx>5AcGL;l|XHLM7m%yoi00xXjhAtn;IRn@hB z$L}*+Pr9vlK`4&_z<$-c>89t#vCF@is@*kXx(C0RoqqVxp@zPVj~FaJiby{5_g1Sc z>y589oVT;PjwuXp(HlohBL7Zzz1oakp?QSs21)JX3;$9caVROWq&{T0_P_4}hA75| zkAL^~VSqM|H|0G^;uCe|-`D*MCBP};p3Hyh)c^nU|A##3_s?F~_x0# zPvvj*1eru_v0DGD2exNxu_n12!f;Oxpd$mIuiMR~Kn?3boAr-Rf`PBa6r257$#hh$ zpZY-a?6Qe!)dTFEtp@rVABOM&&REYh2z?$Z!$!z|95LE-Wh%?)zwDllwD}sTsd)gTl}d%PZDpsk_go9H9A+_R?GUMlz7vo%3Nj5m&1kEJ~IXe z`=h_JF+d6qAp4K!U;X~uuYO}DfBbZ`6&S+uh-bUU|1MgC0jIq8KG9|UPl4d3je&08 zySZCXN7Lker2%~1=8x{TxfOS|{=D1X8u35fuK<-ax0U`j+@8w6zVV3jUnk?7_h*&n zreBvYwr$P>688VkR~)_i=gIR;n;1??yRNkTb>_d``5UqSuLOivImYuK6+f+x=*&0t zrN%@DT^O2<<(8YzGYe|X=0kQmK_(~n+j2Bv$O}oBbd6?_A#ejJ$kjf_6}ivx{e z)lssSHVfqrxTpVyH{-sNbS`f#&+h03Soy~YjPEz5nB%Dw!R`(22=SNN22TzXDo5V$ zr#Sai-O1%U>5C#jX$a^E>Gz@JIp};K=w-c`&L6#x8UoBY5(=cKfYJ@~;;f-C{#Wp` z*CB0AT*}g}8*b=@s^wcjmnS-7xKHN;%pI@(aYB95Wy}9|^v+t)^nGIMr@AmvcMKVF z`LhT%VW5IW(Qlp7E~X9fC9LIpXCUQHf-aS}U9p~RSYpStO( z(Y5=G&gGhJuX03drHzY$=(eah)4!gg34211pFG8Fq7F{Tktkjh_ZQox9{inLP15mJ z{YrtYKRzSLt8#PdyzG`wLyc+Zd-e?n>wD@r>lgc|lW%yIXTEE}&De5Z>L+E$TK0%| zRy+Tg^$kGf@?JhSo*GBTTE6i3HL3q;^#GD&`AYIWj(MJY8_8s-b=1iwf1bFIGrc-u zu5oLz<=y<#yHUjw(BqvcE7GBmv;-Jg5W+Z5@azOFPc^dMu12jQ&xho99Ixx~T1B3* z)T#_?7IeKD2uWL^8xVYNGH03qN_e~O703QB?%qD0>HYs7SG_yVIl6dvMG{VR>L^!9 zh=o%fr*je|xyaZmA!g)qm|5pkoCwunE=Lh^Iak|^*_4ZwxhP_6wuNRfGwcd8elMNQ z`}FzzZnxiGzuV_~;g8(HUa#ln1d@`bP$^gcdyfAhPh+;<<5X(9zz#brxs9Kf7LHX{LYGi zdPq|oOJ`7SjKPc`P${qw@TJ)SDT`xG_O&bd;7Yy2p0w~+*^M78NfP@?R z48y6BrPbvZz6lOQzG{j@s*t0>+D1GSbNIY>r$VWI;A zt{h?^?0gjF#pPT^9{-7&c;H3tEu!cVtdV7vu^qDd*oy}LK{^w*5Bf5PZek#+cSBbM z`ZaB_GSDWVEBMvn5kBE0+?|>Cf8QBE?42J8$IQwc6p=a!OUwgq&$g-`tQ(}!CX%kf z}-e)7KiQs1X?`+KZnOHSpN`%B9#d zLdV;m*XYMJnfXml&vr;teMHM^LEh1?eiZ{Vqdu6*K=nd;O`P6?GVk=}ouH3!dsxaJ{#J1BWY1B!m(`>hcKy(c@=N@@0MpK4_S!&7W8WNQy0EL}d z%>x2!B=m$M9A_Rg))F3D*G6?=)L*xKxQaWsHd}oMN*7d>-Vp#so3Bu zQ&(5}>rHyPK3eTfJ>yY+GvP!NFD1o2Xy98A$*|3JChLyNpoo)EOos77z&j z?aZGyi`tYPt@2+Lk)p2{7Y$bJCh-bgjFB+W_BM?N;wBU|*Yj(7(lBNrimS zC-R>huyNlh@tqm^`xa~}_%x!*yBfS2d+xSWQ@8_Bv`@N!Xvw~RUpxD~%t3W8^^o<* zVd5^GCQGcOSQe|Gh^F3E7AF_o&FCY5-JUVjTcJ5yz{<)jDL$}?-xu!Af7^%?atwo3 z4w7fsA`WqX<&Z6gOVWq9Qtn!mcn{Ug)gtgj>M!2x%JH{G7YO1P#^vun8zC)vpG3uC z`mBdela;pX&UV?a3W~RjFr+*rfn*WhA>4t;X02|X+>Aqq5LE~w0hrDi$vW|%^sdH? zVx*}r$x6WH6Co@PhCzO=jMg`{lJqeNy-j!BNE@IxxCd0(@G1ak;WJ39erN4k+2-vJ zx}(M$4@lZ{M0Ne=eR2Vkqe&Y9XJzEEqNDNQ{Ro3UfFOb)D`@E!6X!ft#&M{c9cm1p ztP!0Hj{23K60Mldfh;c}igslTh<Tl_;O;q&wh~EwtGqK5uI}BLR>2ZFS!RY|H1bVn*$eQEiXK*nYzjw+U*A9>2SyWYMza!xttu?x4p@ z0cx-CmxE-=c>2ukBEVYx{PC+XhWCMJ?jJ)QW%n8}z>Vkxjq@@#AN}wI>|u5ny9H9 z`Y-Q%y&VFp>q`eJ$j<#*g-Rf2U57M1w*cx8n#ozxdWOI;s^!v?|CpXfn=t^GvPkc( zOHzc;pMLf=BszGfz!yk3cY5DUx?Hj`^FQ2UDz_)iMu7$XV)*~RnCAamAHn|-=j6Yy zz)*e0Y*6E5as67T1t6psDJbm@LT1{8=+NM%ehVqn^j-G7ULW`|~^cKMTzk z4nSQ6e4ys;8ncnd>lu6K(xsT$7pfp{=T8Agi7O7m)}+@D#l7-3J=o;8H#bs8Nvc<1 zM3EiE!0Xk=J0}TLrKHFG^Z6eFt=VWp_33{>$~W)mPwS_V#qwJo5wkY_`6$`1`h$^Q z4UFWvUZz!gz0HoagWc3!^U;=!VD|xfChv=e@9o|OWo8zG+hE6>4 zXmEZjWWcHwQzj#!-%KsSM_bFcdfXMGt`Q=}#$dQ!%Wbrf3Gzw3N)N8)=M|X-sA>Q= zDI2JQcOS03=+-P|9|dC0T(YhMcqiWV*)Eit^SLPTuGDlxwTJ-_7Z@)_Z1}3sKS>E8 zhxS99UV`hzD9yCl&Pz8Za&KzP`ce+%u;4>Jm+4ZLz z!#_~C?egY@6^lgwBS7*$Lz}lH%7LdncU@Eb*6^Iv@aEysZnwp#;R)^80(YX6xKawP zPfA62p}>BDU?5&@XuCSt^Dz42OW4JiU;}*9h%|SHP-G%Sfe#^8KEHh8Mg@?odh(Oz zT0HDiJmaYf9UMzd5+Uc5!D;(B^=Y4uQ*_41&h6Y0QM{WW^+V#(Yk+<)crdWr6)m|@ z*Y?ineT6%l6y|D4@kGq6#);!)=7s0m+VhgT8O=MGU*lY0LuEad?{uK* z^+8qCpzdh!&z8Lb{Mhl4PQ$u#OJPlVq3aAR>lFR)W>5KuJlc2FiXR;T3Ty3TAb;!3 z=E;C+7Ijla1uio3?_>3&iQdC~A;WvKGVu z=D4S*Vi`-H07!;3W~VmhL>?|VRsmG#+T~baTS0io2cki^_9y!x!r0CVKMAAE5*i(M zd5{~nB{?oH!d7HylNDB2TGz!4`4|}1yLj^XMe}fWp)gFbAm#G&u7mwz;qEnt1Gv(A z&Y&O8p#~Xo%8okizphP7pE&>=AKBQcJ$t*^C3ZDQbf@(^fE|2Zm~!OX@HJLVzgN@k zBsy7A2M!7HqU zWtl-*WiQgt=?M!(M}W?Fw63sh2fsy-+Kv}x5_ijBqNz<;g3`%x0JUX77 z9`;NxYAg#Ks=myN@Al}e(G4XB@T$6C<{vz78Xx!;Lh=R?8D)m(xOW7S9~ATz)(;zY z7|D@2gH}&dmMw0g)y^=ixhSIy@M=uTqFY7f7FcYvaWzwhS;{^8x2~&}1BzhxfrDg7 zn3rSI`#vM3q}U@i3!q5r{880z^0_741RdcuAUS+CJycM&>w%SBNgglq*ps1wu#-#a zMZW$Qp9Z;?g|_-u4v7MOE$o6#kfneP1UDho{F8km zO+TK5<%^p8Wlq32soTMRXw^KoJA>9!Hhz%Q4s&(!PS!R7?1FAy*>;tFjvp6 z1hY?ZQGGsU&{j1JuxAJWwX10aFQmTGr`x`_MuaB&CVCjQo6MlP7CC<+cLSb0?JQ9VhV zVTLMpVnPK$2FBodf1}_boBEw}P1k>Gy^z^jQ}o^lPcDUF_Z?nSbRMHd8D4JO!7Qs? z;*M(I4&ONEu{b`U_c$UYgrH*ne)m8I`*6AoQ1J^hJG~DMOv9xn_k2y|yShBtF=UE8 zRjB!lFhdfBDPYy1n?Q80km*y&<0BL#uVvii!Xo3Oo9Et9{O7}sG5mYUfqX=%-WzRz z>lVAOPD9*rY)x?k)UX9#n?AYgYTVO}yUQ><=#pq*-A>Ni?wG&cF^Fv~5Evq*X%7SF zuXo;FO3RwK&M#%`+}QY+1|SF%a$WV=E_LI&J)-ji1fHgHBY3{3CUJz+H{3fB^Od3BLgQH$e@r}O zgCO8+pnk{z7CYK7o?I**5Jj0o%SYxu>u}i`1*ms@?zyO^25-B0K~Vtk*cUn`Ftm|A z--TfIBj{GcmoB&eQuM6$M!(T|Gn!N%I8azHl$6RoJ~4U4F>I-;kq>;`)^SSgrSfA zOb5JUM0L`BuMaO&b*DFJ8xA?y&>3-Wdw73H!9Qy`C2QuML#Buurqs_&j^wtN=-AyX%nRGt#*_P%7z}X+3p*OVp!G1i;by`IGKHeOk1= z_EAqg+LCDWpEYqYeq^BTFFh6K(vssiu%Cm;N5h;t$~l_GYjt!%uXoP={vX#R=@-Bd zdw_v`W8CKLQ+}{1?!N!Y-y>t%y?ux$$KMCNUMoktZ}YqQ=eHgD>C>NweS_P%u_>(A z+Cn?xofCV^3osYcM3_{)A^2+lc>f^3xeiQDO7+g1|Ey?Kkfv+Lm8tc&j8pmi6F}1a zJrNEtrw(8^pZ|X~8VI@qQt&^qdY;4DyZwMpJfK4Lwh$(5}ch)?Faeh%r%ST-SBRzA<$kt~kqec|N* znv12}1`G@z@Jt zgH^vMro%3J;3?whKR{Dv2`05gkeA3P>8VOates%`xn9qmj-!}^UrZE!A2Z9`NZeIk zm`k`Yx6qRXAQWrzvgTdHkE*4hlyo~%N9Zv`LAh~>$75I*7*m>fjQ#$)i_@#A;R&ai zCya}jhQ!t0K>JFSO#PT$tP`eQBx?%|yP8l-Aqt*BvugDPApE;}V-ln@2r<4~V~?0` z2ZTB5Ql)U6P)=J-z-F*Z6?HGvs66KdF4(aKWzBbs)`_(78UAv+FaVD+KPANHQQ4)N z0T~3}L?`99S4@3MY9F{pL9CXI4QvGe`+=0E~j%X)jH@(jj4o=9oV2@wH;`k%} z6|nDt8))cGAih4}pAkOWa&jAX^mpC+&!KLaC)C?VCHJ|_%mhe}Imb~NJr_$& zpvAGfrZ!e^BJfFGSz}k~^cZ!h!a&3sr#yLGp ze3`Jr5GVCVBhEQlmtrasPrLYdR%V(3PWdskp^sBJnp9o|A(ZW+Hva78+Wz&%Ca2=H z`oqSROsc!zCVfmlhtG`sfPor#-`x%&n`8HLa(i+{YK};Hptgwk*TG?8^3}=b>T?s| z7PT*qJqsWb_jSIkW=ATs;ywrBu=~mb-U&-HD(8Lb;_NEdg%;Hm0c8Y(yEZlsQGx5U zmp}kibiK#~Cp@!q!i8AboSVhNh&Q3IfN5W5MF*^?hU2RWH5E zuso!y-Lzl(-H8=d?Gg90`Q8v{@Cv@_LO&!<8nJtl{93!WVmN~zN{f<6O@v6 zN^*Otp8@4kWi@aL1v`F`jmaAy&MS_FNCH`3|admtD zNj4e;JG;JAGqtOY%)a-$w#(($%J+kL=;9~4=It744P1+DF4&h$yXXf*Bt-PN`+~i- z_!oodn2wml-wpA?`@#y*YJ+_gZ>gO14&o>B5}j6NT9jzk++}1{QJ`mlEuGuc&8X{* z+{$oA%g9$P|I6LJ2?$ZZ5&ZuW&fx3WsV&{Uk5i8cPtKb+d9k~sk1G4W7Y~h}pHIE) z8w^B?Ka28e@|xN8=!^ULNmqVxcyfUB)b1~54*bsh+5d?TsCmbZWWgunwSx4v>#i+d z^({{%pqhq2YiSSP?(BL10Mg^9g3MT0(cr2kIKA|)fsZtpXoq!~Os?^yGX3&V0nwvs zQ1=H({%Z@EY0ZED-()s`H^Yuo2>k)a8{4ZQGn!KxQ?_m?yi2>Dhz{|BsKgD^A(&5u zSmIt=>RJfry65?EsNdQsx>8`qz&B(`4dzqmV9e;E(@!C1PhJh)#ubezAbvXAAL<*BZ{8XWHP-$ci~5qliPJOQcvIazs}q}Gp7fP<9zeH zHQHQ^LQ}qusq!~ShQ|TAv|U!=&eFWN`xd*y}2R;ob z*soaNb5W-8UVjJ=7`;lloT3F^99C;gas~Jm`M$_a$Cw(A5}=tj6PezLN!hSXn3U{q7_8! zzS1?@ZPf!re!tQ97vtDbHBdfqd|D(qW!pKYA!l~q0f?4bs{~zBjlDMyg|KOOApLf` zdF=uLIE9Eb4Nv;<)rfEleUCF9*{Rq39c_V(r?5cO73vzX5)u93$$$k>be*wCj(kKS zI4F%J1mlZt9dggWpqVrI-WPXMs60EaoVu#S?i13UdZLw5JPn&0+IhG=+=RSS+*Uo+ zehrv>rtjjIJHJXgrvtil_-;fA)Qc49gwo}Zi{D12LVBRZjhy+VwnT4M|8aWcBD~w* z6Z8tst!=4(ENGNP>QIl!_XF?IOYUB*Qe51^WkJ=G{WQJ~nZrR?Ny{^~hp6 zFU!%siZaICubRKAY6e~lXn?h~G;BS0vwIHPOIK@=s+DH({2q9|6gWD^aojJAzl&BLoqiFH2){rxMPe%n3v5MgTkV)hJmbwx6tK_xG^v0O+=t-y1ScE_k) zNtO?ysz7O?c7Yf%VTe0(ylShYLLaHTIw3BemL}z8?Cok9l)!t-Ae<&udB(E^Q6DK^-A#T!HYJwat@i2OIWJDCYnr0HvUj`>T-fl zRrw}zij$+RW+sF(NL0cevW6`|&*g9-9 z@?Fx~^vfAJ?2N>Q1}*Vr-TRlM0E0VecYf25;L+M8{xs??#Y`kq#6}k50Mo$+SG;PL zI3m5fBAP@FVjp3P84a$g@Gh%67KGyHt5c_CdQ?=Vfg zuxoGXlDQd_*oDvi>XWbpG#3E#A?4B&+LF&gOk+qxPksBe3G%HONoaI&c|ibPK*+yfpYAZ!+f~ zIZ=j`{6ZFqJU2XVrsB40E#*<_;G~EM7twXMvMuy!?nj1+6+>^27{ME4ldDXo8_#;j zhOruH^^GeN#}W(X7#4V@2Qp(nb)&Ow1PHj zLyyv4m}w61WdP#iD(U0l0W&VxqRM%b(WsOZ_hvEH{kPyZ(mR(#GK^T-q<$<4{p}q> zp-ELPRT1qlVRb%iB{VyZf!f2$U}5qEI=quupEO$GN_O{N#)L_rj(llhAFXcdU#ckx zP}cwqi{S)0Y3SU1bfm92)5G&SL*qC?*qa*tBN}%U2Gpm~HTa87A3@W2EB@{L^l;pA>$om&>L&#To6ruEr7@y_qzaGGg{&{GO!ud zp2yX$z?0>ibosD#zg#~n!r2(+!WEw_6LdrJU-^56eZ-d3Y#tE&EBP^)@kFq@TWZFy z^q)aXhw2A#g5y;_oAgGzyWAX*p27wl^8U&{7&ubbFEA2`L8eSk%>_bWuRS+(ez_}1 z=r^hJr*Ex5ANKYcV0LLB#$SedzE?_0Y}uMj{gqvjKxV_-c+Z-ieaQby-F!|~$$rM7 zPg&1GAEg#6jY&DHOU=}VYOU3mRz~vx)YSOA?m~-dV$w^15FO}jlG$N*6?nw$;m?W| zuz%Ks9kO(+#;7oZKR@9*k#sh9@0y{o(KzZ<$XjAIkO(>X)fz0}YS3R+hT^Zgs`Ka0 z)iI!92+nv63SAqQfx%xpwdJp!`r9)51kwAyD>2OYmwi-DAJcTJLjvS&$?n=~a4Y4T z{(k1>%4G8OmT(nk$A3Z$F%H!Lyh4-?qFW7?H~`br)lUh7ICTmziZXf`ZtGQ(e~j<*xPwkF%K@-l zj9b&Cy1NYW4v2%(Sdve7n&|pUCO=Ofpa7MzrA!|nYa6sq{r*2@y~JY=5I5a6(}@%x zYyyglM1=v!h?(06dOE&o92-YxhVxozy(t%GH|*ZrdboCMRSw&+ikUU06!HCA=97gW^${PJ}G1^i&~PDBBn&8u>K37b@xSxsh@ zlswO9qI*eA>K~012CVD2@Wp-`J_YO~&aYqm1eo{#JV5ln=V1TG5vu$e3b&>n6$)m3&Y-wr{SQMv+niOyyEJh;HOA<3d->_S(H8$pT_hd0h z_m{0UwT9T(*YdtAZYR@vs1u`-VYAfY@m1L`VPvi^ISQ~n^ucGqLS1~aQ=!Ew)L5u8;kxdxDOR4M_=c- z|7@U-(WIwDv^8GGsvyge`9l~^bgT8GTkopJlEn{R&^5XylP$L=Kbx_DJ7!%u&1?n< zNAMtP$O5YxZyh5Rq?Gq^FnM+J7|DT>SfD3g zo0<2o2B5)hg2YesKG(sW!bJE3_(u0C$HsAm-K3Emb!_|i^*t&W8z3yZ29b7OAp~)= zB88kkDe_mv5*}Fe>kn0w!%AL)TT#X`KqhyZ?Rku~om0KM0;1+^v6;x3EYvu8F-@!! z7kf6ukJ1pvsq!+4+t6>^89@@ENe^&;5Y4q$y8Mb;gQsGd^>IE{vj91Gx&V})I`9y# zNxU*6Xyxl>=0&$Ld;D4}t;e$1lS$EA-LrIzY;u~XYSr5d@>E+5WYP1tJfeRTL=gu8 zdS9Kqs%$>YO+#^vJ|5FQiiKyr{*BKk6!+F9F+zvPcj{Bo2xXnM`hj4ytyFqp8X%Gm zn{P`QR4iH}6G3mWtJt&spsLlNm^prsy)A0Ot*$7w=;07ysAFhwXkth9Wc(=Lh{RP!t#0fGQ2C$0 z=$QP5CU*P}#A22xx( zJ#-^kk=1G-b;1gLzAP76XRUFTwGzi^fvG1tJz(T%Q+`e`~CJwcF& zE>QM?r!$M&Y}zPoHtm#ln-0qQ^Epvn(@#L}&2|9!!sEj8ixqrbra}95&i<@k{GI!g z%j2cYkZE#$uG03ydUe*2ebic4-bSU2)t_w!>%%i3`ul6AcYKabXaRbyMZ&7{y8sY% zvnUE>_}G+I67BUwV(54m&=n_ic=ZmkBsFenrGBRALUEMU`LuZzOk*nK7Fvd6K508? zN!os1ygztu(UhgpD$t-HjW24@1?Q3PwDIH$ z=>QKE!1u6a57!edl5OHLmO^qg-LyDfz9MQQ4w7p$I1ueH#p*k0lx4do+$fTm!Lo8p z_t{m3(0~~te7wB|#xGk_*wOow!)^}MXclv8)LJ1;6Dw7FZ;|7T%6;AtZnV!m7BsW+j8x>%stbCNwZoO=&CAOtD{Jtu zUC9iU-LKv4qnv)cQs?X7L5)K(@pRqs+s$YG!HeXr?d)(3(22Pg?#G;M8?2vVg8F`2 zJ+=a3Z>B#3$hq3E1suv4c+wQ6tAx(298<7awOX}4OVX`+vd5!2N+I0aT!PV(a28)OnL4`TBvs|(lS zg|J?kMM{_n1zFQzisp@lAd$Wpw+fso#c%jdxRC^5d|Lx(+s1e}*Dd^7iVB|WGG&OL z_z%@+Ei>`Ge+Jc*EuT4yh|UTlh-%}5?Ol+Z!TBRNZ=_O*VGIpU02QKTv`XtZ-^IDb zk2HMd%IyvM=!HqcCzaOJI(U2JR%oP@yw}&fe_}It<%(vA?z-JLy&U)uMIW^U0^9~0 zG_7LhdA;Wj+IJ~-d;g>nMVDV~z*Io3S7E~pPYAG$r0o>NN#{mEPDlweq#&HLuF zt8+rRr%;|G6lO*DzFi>8;tIr&%BJpa`GRmx8m*-mIx8ZW6U)QMdjwNWhTZadd~Xg5 z%S~O99p4ny>uHe@flr9QLBl5wr`m;79HG@OS{}^?VuxdnK-_TMkXE~#x?pqS!P@2x z;eG6xGN8ex!aH>rugOz0=+YIS&#Ld1g$z)iW`=c|&T#bsII~tC&~hbV->*9aGfyor z+#&v8W^{e<+?j5*+@~>*>a1)`g)NWmgN99Gj4`eyIOt7yxiPL}wton7ZCq80fHR>W zuGI7}+eex0AL7@sKsjo!N&ZNT3oB*mp6KnMi?vQ`O83#$RCWu)$Y_BtBZ4X@s0|rr zFA3qPZm=w=xTeq@GM;)03xBP{+J~-w9vmE}SUN%|T4L6x+N}z8YNwrwOZ8I0HCrpC z>ZMrZnJ!<2YCx}rXuGaT7y%{EEAWh*A?iA;irTraFvo8k>ZN|vYOhK3R;O*bTAFDe zVm0v^MkSU}aMLRk2r~bjjvKOfIE$D|k)wjVR_6wM_*J0G?>JkfLC!)Y=Z(BnDGh}A zUJ*)ruY;GG`CZ9cwfPSCh|R|mB7C64>v^)!pCJ}U-vAc~Hdqm&3Y1&z;;xP<-X8^g zID6L20%U1neDXza>>;mJ8amf#taBloyd8x=D1 zL8Bh?L5slUA6sSk-CU!^)RtEN_vAufkp=MX+Qf2wh3kN>^&!j%Y*W1R&LLo z;%zBjy*4g`S)eK@>Ulos4#q4NJjM5Pt$C1TYjBx(P|iGFVtkxf2)x2eWXi;YCx59*&PicL3FwR5fqy6Wl4kD~x8g z#`JExdi=iuiY^0IPPS~CgVkAGV)Qf5Y&o2W7>J#dqp5m)UgxbpJ1-fJx`ciFhU*Az zhB-59!Y(kMKU6*6^1DrndARXS3!|8=%?(~8y=*KYW*z=%Z>@c^#*K1~t4QMxjK*X7 zHEUjW$5MQYNBgc*%_J(b{|U*ELNK)=UvTKysLpcAC(KPXzcp@%bIH zh|NrSPh@xTZH?ez53+iAHqN-f8uQ3d325~pM_57wTJ?jC60rq-H^8P{VrUD z%+ki?0EL!KZ=4b(sFtd4Zsy)uOE9nH{BKXzB3Xp`2<1}vYKbkjYLU$q9_G5#O@g?; z{@}Ts9=I=PdsgvwNS2Zl2D1v0-tiy*6U5fu>V8?b`TfeYyy(>Z=p}*Y$Vuvzo3FjD z!l7M((2=T?-tzzuB?|mW$h-oCOwgH93OO&SE*Y+A9#?%$VHL$aU4v#zX_DlKtwWqR zjW=PT{W62{n7vjs!fIp8>d{MF?lqck<|r-bJxoJmIZVOQc7?9I7hhcIZ-=@96sGao!7A!K z7;Mug7vwCp#>RSlI;Za7rYj9?CIEXZ#ci+ngiIxg_ep(Ij<2cjRe83m&YLAKiUNt8 zL3t>FE7E*+Adst?UMA9wCnGAFS6@}&R&4OpJe1~=j_)giJv~*=lEsQr2ZE;4p#%608 z{7FY`(zxre{5n;UD)MqurfTu{1KzgNLz_&efi`kNW`86RSBEwQ8ov2#=3**N#N)?7 zrfB8RaRPoWT0BG~7Bx1TC;_F*$LrJ^Cy!tJn1v=}3k11Z1LL-@0tvR|mMDcdYkX@< zES!Z0q5S#@k_lrr))4Y)VF%3yawp1Ar9;U1fzUnmsNMwL^(P|^{YI=nE$ZMGDFH+L z4P3tuT(zB|ty&_V^8R-m+d+=0lwwgz&4*Rs#GK-#I9xsl^_rYo=B2AyYl!y;2Pocp zMvBE8AHN7l9S{z&k$cJX1Oa<#m?_FYYP5YA^tVSq1aB=@+R{hxkAIhpq&xE zoS&WRP9j=sgn=$)>6Q!|Yo{@uh{l4u|T z(UBqO79!x$m3!lyMY`+awR5h0rQI}DqjwzAv?SBcbM3GRFTv^NKcxp~)*GJLMbBBx zp0v&r3J+&^tu6`nGD2fvprl28x<0Ye8ajaw4j@O8aPpU5)1KypqJLi6a2ADz(c;n; zRfASz<>s~QykQW+xZ-3|)p=Jnylc&{f0sJ3qJ8pB!32Zn`UiWDsfDX>9ua!2O zIO9ub+?%-I%PT?LDBZWpX7n3HQir1Q7pd~sn`HF!=8W=YzbTaQTQLVmsF+Z^TS??e z&NHH*F09^B*0BZaAd%vYW?fY8yOWW_ixbsVr*OZ9Mc0wHtgHW=+-~Ws_hVE^UPUm9 zj49JT!@skiM;AzyQg02A=zKCMT7j?-2G7Y6VOUtCF5r@wPZNR_ERGP#0md$^tbZ!s3lQP_@-s~&|HypLd?$?3yEXQhF| zrw`>v!N<%lb!_N3+!=9a2H>^%N@G@LgB(y?zFPL5gH-F?Uo07Nqh?p3q~&!*jl&Z; zG@TrZ80Z65)X?4&3iL~lW-Eg287S{Npz(FyJ5sZhA2-TNY?n${iG5ycQHxfcQ)-K7 ztZnE-CF`#N8;+*0d3+kFrR>`7jln9I$L+D6_F-PlSN^a)-G*IO zkI7Q`@fM*m{;!5HpK=h=gZx`Z1(^}93ncrj$ zo_;)OPsYq;H<|+dfB*;k6L#UKCdfSKJXkkI4;9v)7#I??Ph1J~6ynCbfBWglL;&5t z-0#nI$T1fII|#(^U&mahRY{0|pJo{PTEY*;QA?td*QtE>_ePGf>cae%dJdP|PzxJ> zg~!E3FiAcz!fs;bFbm*{S_^IUmr!Pv_1^rvp7J#y-JirTd7h*+*!J;C_pb~;I=D7= zJ1&GgIG4Rar5f=WV0XK{4kn(Ev=OP;!R5EQK6JpTo=~M`tVo@qotSZTeWds2lN+wE zu4JH-&?ulVy5+-3QW(NRJ~SZ_XvA8BhaU5CqPoZNsbOpRXdq1c3D9AX!mboWAb@_0 zzG|UKkAm~VFEx&H66$yjS)Lg|BO}A_t7v_8*txUpno3iHA_s&gBQqG>I5#pcj+IRS z3jRPZ$yhkTgXRGXRNXLUbQ9#U2xTB8)115$s$-tF{7wRIOa1(|2W}^0WN!ty3vR1D zq8s#(q;$c`?AHDiLvSsxA*cuS9h|xRqty|MncmT_q!9z_DzZLg=lrFm9z2g`ilA%lJV=hQA0>sV7y@-2(~(hD%YSSD!m4V} z#cZXe%j%NHD_lNM9uXm1{G+x1w?}r$m37q;A+zoWZ3(Zb&d%V`35&74>i{&#S}62q z;Kry0_f~oNw^kAzmuC>&vHUdWzD7};E%;$l+qka9mDors_B}VotoD=SkO^#?L+DG zYT0kf%h^VcNN(~Q==8yk4R`K#ttteo!-D-pF{;nC>5juw%Tuz9qN=43t;!a6+aTfN zzRu~;#1QPY1L@p7v=f+q@XK=D)N43r1Qgb)n#TMpPj$nf^{iQ-eh+_UnP)CRqCl;F zPFMGGEz=cC0mg+^!9wYbWk)xVKMj_I9YrrQzp8BV((xQd=PSq)${wyP^0<8KRS`a# zKO;%j4RoZ|&tn1?&pqo3^WeX#tPDHYqp9)_+|}KEZechWd+RjeUlx-U(QH9qN_S4( zM$#)gr>~!%Ofmm962BiZau`?WMb=UpMFBiZE0&7Hgg%0{QRIfCBH}YsP+dONi!gcF z$ErO$YZ2s=|MRv|YYM5VgKzg+`K89g7%xLk_s;wC2Mc5iQxu7Mv4_}VropVPVj=c! z0ko8W68dycBmBTkf+;rP#T2dB;K)GgKo>!M+|TKISg@k2rb@eHI;1p~j#$)8UW$Gb#cs^E=p}{5l2%6f zSCvr%wz}jQS)@GoPxhQt`D(_^r>Ti{^&{zD)rg@+7{a3_2t~Mcu+KQ8r;W7XVVzi! zT5{Gl zV#2%{L&B~!2(-CCK~9JtIpy4aoaRCDi)ck+a8Wmyy>096635YESC#B!Qm<`A^<~4E zcXada{f;h_@7@{k)0p@2CkV88!MzeD;oZ3~H~?>@f5f7noColgO6mAfvWklv6oR_4 z&os^)Hx4BKMU5WW8(+sWL-tSc3-cNf=F`2f|m*vz6 zxKLasVtB(9@z^^gUPe5wiNZBJL$N7K^tQ?oYg{$1^`}+UbNQlIn2Mj`5B6YBB&d{` zdxd&CH8v>_+;oREK$-i}-z%{aDICty8M+0F;rqoM@`~Y8iycx&G7MV`@5Ra=nD#L! z%{Ji2^n)`g8HS^P*l=#h$@0DB53o2yL6&-OC7>C0E+J5C-22#Oy{htUDm&BMA^ zfu>a|6Mp;|p}%m0#xsk2d0Nm+oMcIIM=R(R=KUe8fIAysuY5gnz<>O0?O_z3IaM85 zT4hs#)#Kn37$uWnCJdlzmQ*y#-b$V|jjbC${`{nA;+39!{cqaa--o&@_s10dblHXU za3l}c8;%OM|9L=-pQWbsHatIoVKf29t@^>aD8R)ZhgMO|dw;TzDw53w<$Lqudgi4< z?+X59`!4SC(vg%fU$3WDsG6A}{0 zv<`{;FL8Z}IBE$<|1hFo?@Ty{!Ew@mZ?`h@*sn;~WM0x_Gyf=X zTTtJy?Bmuxz}7D)`CemVCWci+Qzj8esU|2Au`?y!kjic7>|vml@}*Mbx=km5a&`+V(g4mBoF;~fUZkJA~%RHnI+#B$)0 zVrgo4Wy12Zvhj6AIRVw$^I~M)zIBkDU9f0%BgMf*dSasI@a^V-wPafkH5@_s>EJPL z8)+ARcQqnLvLoXLJ#sCCyEzK|mOOKjY;m(0`tjLP97V1i81?trq+b$Dwl&}&!ZYeZ zTacM$aV%zBe!rJnuOjOd>)2J!X;iFm&(8);_=IT!qTGF@>!j4% zG?06{=*A>wHhw3MX-9Sf>aIgQ5hL#BqU`I|0OOyEz9)S7Gw{jhuodKmb-1n{s#Xi@ zeH8KG-S&{jhA$nC$z!LSCntjAjKOL8iBVpuMM*KmuSc|v&qZS``<7Vx{ey*fEfi!P zU?cxPXgHvA6e=CFUtIjCVaErd6~6}~RwnL^&YupFkA*A7;ELaaq~mXfFxNQ&{v}p6 zHaf1pi9AM$ChS@nfjmJm8)Oe>aFSM(?=sw=uCE^Cci)@q(cRf;IYkdPMo#Q4MRP_l zW8%SunWgcF3OLMJ?Ct{PzYyri_Q>{@Fc&rH+3f<>S-AEmr9z=tDWFJf4h{@oEu-O; zi?31(;hN#gU*VdX+4!(q?-^&MW+DTaIp4VD7#@(GlBWD-LvY^qE6>|3)%km>kkHj_=a@4uSXrc8lb2RstIW$GC=yq z;_~EfK;Xv<&nPL9ae$!yI{WU&hNGJeQ|uni3=J;K7Yu(55MN-$>g2l?6}R0|M9)am z4dYOtfORYd#aobBZSg>bA-$9n@WY}_?!_eN=L=Iu3=u3zDE4tCU`qK!A86GK9`k_$ zT$abA(F-fz!;xdJ1D%3$onGdEAxcW{Glec^&{Cl?QHF9dTg5#-OZw4@x+pmKRK{of zw6}}Leh^TQMWl$)vkj19?aROh6e+qgaJ6ar@eD~s;MgW?Fb9FeXk7y zPB?zB_RV7ZAW5CSw;GYv4N#l5os4c4QJVrprC(81%ao3`vq~3V6jr`XiLow3aHbV^ zuSa8rI@f@3JZ>75-*?ISP?is4GbCJWimMN3#|Cr=iHkT%q-MX46K@GPd{`t?RnK{KFn(N>Z-XH*Xr zXQ=^U&S8?gCa0fev@9R)^D;u%#-Ns7Y__;Tzx8LAjHY8RIyqV!cqPaa*hVB|fqJXY0aslnH&mS+OiiJ8w%aw0mse}Q(w5WoG( zg=-Zau}R0Ebq(eX?=1=uU?2qbJlhFC-nQFBPWXG?dG6o_F!jtHBoE`tQC(Atk=I`< zzA~`3ufNM^2xIFlsSA!BByH3mt(;QlO>Jz{6h8K>d%_r^7Tk3^+_$UDj;bkz({)QG ztkP$%HUM2FNa|}vX)Pav;@H-6%Fpq7g@UWJy16(<#C=+zub4DJK)WszD(2<7iE8UyiUfy~8Au&pP`29v;=ihotfi71En}Bu|f>k{@ zn_}|GAYF^>C9CRW5{%{#_@~dRAIfhURmIkAzrE49`TZu9>Yq?k;M5!TPWzLvCp3QJ ziQZoDd~C?ve6Fki!p@e^)dhXUAMX;U7T(%NW%WTzzBkVR@D=dAWJAsLv1^gT1J|$n z4qGqn(gZK*Q2SaU@Tu(;D{ncs;p+|s12t#!8|xc?b_)Kbrph;MJ0u^(!lwg#L1UgK z7pP6cyL_6@t3SKTaQC$UWRM0>R~Y-L>gj5Oy=WmLkQ8Ko-zP9evI4}5FRkTsmvU0Q z9vgNg10=9&8DY5j9Md%z{)f9PoleZtpAMyugZbZAlZDGG<8?9L!vS=s#3ijR+ox8B z9rQg1=?38j?5`GP%v$xkc3y37u*A&95W~z_ePAFzBAKT>Q7q~Klq<580eY~|`yx97 zP$}66Zkn~@qkw{l4UjG(A|N0jy{RZDAiYNSxrCe}=O zExVS@?F1ZfB}^qIe1UtmiF8%WUZwYxoJEf-xECUaN0&iyi>}xKMQi2vp~*}$ds$_c z*Vta>2UKoJESJu#CKZx8Llw^shl!;NTv3@{#1%g*@coBKRN(pb$~Xq2z*~Cmfu3?@ z0DX5yp=giO!j|28M_oYleL^dP9^G#8RQ6B+7M{-*mQNyfjB!`IVr?e|wJV@Sr0prm z^JuY;xncC5NMlD9GC?eTUURhADgysnu`&9!;qYg?RVXmKWHsriY41JjlFaJBi0jjy zE(f%%XTWtK-QI3KN&O_+R21mzUCA>4PK9M}H{xtEz08YubAZ zoF7#NLQm5)Ezi(9Mw~L1wPSHxPrYz8c0z46rYFD|+56q)al=}zt63UViHes5V%i#o zdcFQLJ8u*3jz$YASE^Y3({{zpp19w1QZxmo$qhcMGGjj{pDJE*_NhJ`SCq6S9i{1M z%JqQC8M$9K$dg#;+w|x&_2q0f*F5K~LTcL~-J&^UvR@c!849LKu4Sv2mGPd_D`~50 zL%#yfk=X8LwRznw1l9>29?v`!Pz8FlzZUA?rQ`S{f*|uD3Nh7xr zP$i3NmADLi8L63=#Ircg98Tj(EUVxD?=2v4WfTI{ccI&Hn^wS-fuy&sD;e*!ku(gf zRV5neE&0MH60BnM+n5|^yZ|mh`@=WIG*oUM>{U>~#f@T^Jb+e(YqoPV8$Be_P>jvz zowv)};U9ey_iCCL$)aE#8>k8WaQD}* z2dY2riC-Sa%{W>NRS&f2=}e215D1##udBSmpTnwv!TcwC`YPkpwkj1GQ! zqZYp@g|wc;>kj8?;j~5ikBBjKkSTHzX+6{(pkbD{35D)-x?|N;OMDf-NZT9J2#33 z{X7gs_&;FZx&^?CON>tU$7KW5>VNg>Al){C-@48xXWP(Ob=y~|FD zbwutgXUIyN5WcD!3s_{ocfb}X^~1&4bNbn*elS| z(Jw^GZo7#*){C^uznXIC7lYD6)w=={jjS--$EBbCp0Njw*6PivQ@#$!aG2;s&_dO9 zki)Yn{NyLJ!1dPilD(bua(O}PCEd&!1I^qp-wpylw7l3tyxmz~(qp|QCFC3*?0%nMgindpu^*eA?9Csv)H$2U^s#*mayGpv1~3Y}S3e%`t*&fU-2c23g{Gb1M* ziB&)?gT*VGU<-oZJBnPxxN)c7EA@ms-XDZGzF`_ECpsG8LX&yd?E5jPEX2{*5A3Ky zkIb1WHiO#G)ukIzhjxxKKbaCM59_oYdZLHvV-ZHex(Cb<%l5nOm_xtQQDUpOW6Wmp z;d>|K;IrMVDYmYLz?tW}MHi&}x50#@d)r~4$JPnP=pHwUzWg*mxD_=cwb))6pg7?> zatAozN&W01%?~?Q2a8ZfKl_hDP$us;iGW~Tli|njy)1~={s=bm>hyPJ{H9+WGaFip zU0Uouy|%;)y;J@iB_hQ{q@3QVDfvfbImH>Vj4LTSlK-S;B*Ab3xld{fggSDre~7(3 zc{9+_@Ac;!55fOPy7%WW7X>%#F2A`W7G@52udK$o5NQKpZ=uC~g`3ER`s+-lO3{o{^^8Udj_R4HEE*z7_C4En{D04j)C(b`PLpfFr z-I|Ep7jTT2!^_{cX&nC^Hv{`^m|w5xw;SI_OSYr5&W%y9Gm0g?rHbH-jK;{p4I!## zQ<&50HHpg=@e^Xm{N%aEa5GOQPW9|!oXkMu7TBj~ux;WW-@bCNrvf#vhIZu$)%5?B zo^b6We!a=FdOJ`N%Sv}RDyD5RjQMmDHEef)(ih^_KDKWp{=_7-zlxF+cw!cfDcQzXOR8$rn(0&+Q+t9#q9%Pv!pTUyvGAw zG^oDv>>EuAoQXw=-Rj9(9G)|CUORl)kBc}5D$A|*dBX3#l**`?p5r6{-o!vKfUH1< z|8+fWt;Z5xug^M$`;DYV`LGf1u0ie-@$~<7OQT*uq^6eo@wNVg*(0LYS^%lYu>DDs6v_S z6SGV!$4nQhypDGh4>ESM`a($p0;K$lHHW7_r_E==e4#XAyQ6D*F*wq?4sPJ6GZ?=# zjvmd$;wm++439xyCQW?D&6`l72R8Xj&}M* z8vDN~WN*(=dviKw2W(C9G-~S3Z0HhB?|IUbZ=@s&B-f6^hVo4%h3RD(K;vU1+$SgD zel-8(%lmEq(wUoD{;CJ=mNNyQh;erVp-<+UsKCPC>y|+&(tj-sZQO{(Hl+L%`6`O_ z-o=`iuOhkV8YumZM+CQF7bfOTz7wy0HC1-r`tp3gO;KBLs`esH?3 z(faoPCxWMf0`Rl`$lNED?nS&}4VrXdVVA>Qq~3XOc#a}`(fZq0OpmjPS1sc=qnB=0 z7X>_%9rjU?y`wW1Hk)a8j=_rg3kDzRMo+2pD&ZDLyvkuDxvyxPl$`n?uWfI`;@1bT zrS=Puq^(dkoA)g;nV9V@b!q7v6(0 z(A-6FB~COoz;)Wm?;A|oi2oBD_^dR7o}4*OU7(ekwJ@z_6Mlbu}|-d z2I(VoR?0amYEMwDEVWy* zc2&Oqxy_*WFSI-1fA$aBRTW0BH2KR233_W%qear);amPfBffe+-Mf`MVV{dEOgD-v z6StCfO%_kz9N|KA8+LmFcADiiBWzvQr37!JusxO%o~<&Gl64+yJthpQ>e5Wn_7{l9;7qlE_v`*J-wC*^91FT0#Ug~{qKWhI}l*K5QKG$GzXxCHdg@f>VHf! zZQflOGz?sx3{^t`QyWR zA@R{`n@b?y8sWw>^k$P15VtQqs%}pA*(uh`e^Q}}qM$kGJY2dYBCdxS-y6pg%#ZTn z&``!4Bh_oo_84y*7tITPOuiXukkk~H!1QBu(3IV;Pw6?H_2Up-v@noY`?lNJTy*yK z9ycTFzM!Fh4TxRez-ACLkVQQ5NJbxUi=bo+C)=m2vWMIvKSuc6fWY?h`gm3lZpDCO zqOu2zoXjU*iPH+$8n4`#5xV1bIttK9AdnllW8jU5!!wN-Q?uO^pxh67q$@A)8FVT2 zYiz2@t%WgbWaj;l6@pOXCVI2p?jKL`KO?>WrzmSGYWzCA;*(qVF$a}&OlD;Q|3G;FXe%&ul zO#{5}QF0B~7Ve!J!a`m%x?Lu>;&ao^9UTihYsfoHsgrgI&G)KXnBL?Y`O=C;`-(J}?}C0O{rvd})jW1!u#(mDB&E0t{DF}a zbd=p9t>o=?tCz$n2Q zZ6sq}e)+}Oa(J?|9NWL6h>J>~fa`^Jrp2D*4tLp2pVY(x^|Gm*VMX`crW(VLtESdNwWqVb=we3Je$yu_MZU;O!K!FzR; zj|4fHXnZvP1y`KeWIFk>uBKYUEVU+jVd%99AE1cf6&l$FGdj9Uv*-A%o81hzRP%$# zwh5Wmy^PCa*I=&H2i@Z$_L&|Jk0;e>37P=Dk8~!uoy=x?+i#RV3nGkUnMODEu4?7- zZ^Wj(Gir`V8Co~My%-!VF4D1I%wk=WSo8Xpg2`ftW}FbhzZi`#hpG*|SNZr2|J!ha z5_q2lN)6)uu6@5-?Vvt~JotWyp((<;TiOU%TMioui{AnZEGw;E|2X{Fq}=V)yt4b} zkp_ZQmIEtI6cw2q;r?Gr)8fjT8+<}sSfudCo2{_}u-ST8OCUm-AH!DrAx;p5=ruft z;VX6KHGQRg(HW7zw#P$Ob{%^&AYt;|7x;<2cU_8P`n%ZcR!VY;qE9{de+3M#5FAEW zn}xwO6#_N#D-u~f7iTR46walZrbx+U27Ll%Y=E#P-JZEma=C_K0H5QoW32y07=9pN zS5-8^SijBAy+|eWZiDQor?x?dy4vk2^r*VBXYzM|18ibq2=F1RKWf}u2+aYEMfnvM zXWew`ccUphIr;jPCW`u7+zl)vG;Gdsd6Wx5$O=8TcoB3&7@Qw&<8Xp|oo*!I=)347R2;0{}V0iFDrzQ#ixjC#s?uo-}x%awaM;wMFAgudb zR#~y;KQ4@NVU~TfCZX_G-St=PrG6i9ER`~ROmXWIr0vsq`EVptisK?#4PNPyYPa;B z=kC>{83*B$z0)!4aL2a=>EM9aTN5j|k{#%0_aaGj>81FiIlA6(w}@vKG0PvN6i2$< zo;=5eSDJVkC+!}t-|cuLn@zjek^egB_soQO{5@F83XT?J_eOVxfS|4&1_8Frn-3fc zqRS^eqclWA$|nQ?H5ebBEXuG;b*gq-p@CRl^JQ%eI_}M+1 z&vW@e2_<7yJm*)De@Pf2iw`0mos{DEcbQ{jBQK$bC1v09<6u`7n^^Jh9F#k%9Pgnd zrg0XpA1J>&EhFA08uujQelfcv!+#xWHkz|bLoX{KlOeT_f-+BftsxN0Madp43)|q~ zt!m^{gTlldR&)LM5WU|9!-5_|*BKESE%E^DWD`DGH)oR^B=7UT>Zc809hj}klzOnR ztgrq|(!g^$>_J#~C(B8E9*~Tn%sGtak2m;8@)Yq}`fltob4BE}-Yb&4yf@|8;(!eJ zaOj*(o}uY;rdSDug@BObcd9o%2mVof(#d(Te{>=idL!R)tgNEv`1Pv?w*Vc)vtM@> zU`^T)hvt^me`(!TMe2^g0*m+H6}bEQZ~7Vz%HH>WsmofEP>N2fEt!|#360o3LsW5n zF}FKVPNrB$czdFA-r&H?jw!nJXx*#8wJ2dx;&QeZG*piM8duWdL56F}=X#TCjrAFIpjF;+Ob_!{;LEl}VC^%W6GcP47is zTVt8{YYmF(W(ALWFmkugcESr?_OZz5M7Bw8sVm7*r5sc{ZMB$AIebc7By1tT*uD4K zS$Vee;VS*`)my^^Z`;@WJ}aO{Iwk-a@;IFBMM$#<_mdJp!acp~U{pq{fOIN*cf^#}4H@MVwGM^#MaRsLGPd!cL zuAdWztMp8)g&gEBCaj$jOrHIF)FZ04HL z2Zs0^V?*&T`X>c2WnARO#ZEM$wV#Z}6>s!yNHUpvajsdmOr zsCDTGBpvh|`;5iiIIRe+gQD%#dIb%Qv{!qi?Ex|COZ-La#OA#^V||Zo#-w}?mNO)r z>qfoWOTqYf|MQV~A1d7o1?2D(LY3m|iSS*qVIUsb-$R|%+@zaBeKR5E*3P$xIeo#= zT<&brYH@M0P@;?95fT;Q;)pI4n(ZCV!SA9c(;qxosN?C}OBw@wdN8Lmk`s~C)umZ< zA529i;p45k;q%+i=P+w-ClOw$M;)lq%O^~q5na8te7^tDz7I>GhT$b%nH+Ooinlz0 z)VN0nr^Qe7b7|HxdZfM;oLEn<6?7`r?$vS@?hDI<}0SwoZdOSpQRNoPX!0c z$L2KKzTU&c*Lkd+TJ6iEPC=Hh<0=WpN5)Ot!@b~#diOqOVe|;4h4_{liW8`R#Xe5C z)~xh7mKMennL8|ySlxMIMy1IFcYmSV)d+ysNh< z!IemnZDGOARXTl~gfNK7h~9N+&DR>F5)wj22l@<{={SxO$9$|SSP)<1&pFR9Y~BJF z*tV2ejz4m=$9+pM+T2bN*+m0b`r7huRN|y^QH=^MQbK+@Un*2P_kI3e_QI?`Qd8b} zU7r*&C@4nerPVA9?G0q?{D^DW*$d@5{*jzZWiG6FpRQ#rH!>KJ#MaG}qW^_h8Ft9K zXnRgDPeMfY)kPYc8b`gQKREl1QvMX3&&7#m#ok4|O#CS12sfALuqciM zD2?)DC=El`E@x!$e&x*NfhY&^&6pHA`hO4WTPoP08%Ta<;Jd1#pmc~ zEh^Ynk*SJN)~mq-898q`0krQ&+ay?FRx3I1H_r`^x zpZXZWJ*sKSmG;KyD^@4zH})s4@7fJuqWEAc4@EV6dsy%`kQL}J2eIELI_;9~D9%XC z7xBjHx0L1LtvPoc#(BzFnDtgXtmJ+pr;aYSPfpK-y)4JO{|F0ZZ4Mn5Ff%(Yl{#&* zHdUlbow^xv2RQ>#rKZ#f&&SR*niWaCo2D^G&~99bz`X2JQSe8Azk(6q1aPA4*9IHt zH%2|=jMS1Ggaj$Fe<3CMZNu&C=_78SxQAw}hh?0Fj0;;67fK+f8H_Odc)|n)?D1m! zy@k&B#sc!SpDV#)bx*GB?R5^#wv;@Seh?TK_>Rud+CA;I`JjY^7tHi@+dg|9$3K=E z)Sg2adCoRI*C=%BaWqTa6Pig|%Y(vJnzPZSuO*sG?E+HA(UEx>*aR=D41iRY9$*7mBMsbG>WkxiV-0B0S@WSls>$;lP?dZY0=JG+>8 z?3#Jub!f%mgc5c|r`@ZB5akx}DA@cMGK^5pcMc{Bx;MG745_Wmi+Km9ZrcGDL5A=%_C;`pwNX zsW=;XQqy+yP|tE>@?;}7jL$trT(bPbbf|yIbA}F58!h~_j_jEYmS9LS`7)?h|rBdm)U(v*RO$_Ypg{}a@fyjv<`46%Qnd|a>Ay_X@C`LOaf z23e&;{i9qNp+j7xo_4a&)8k z6p%w07Mm%1b?QXqW-#gOD(*UZH$M@%oVYv}YijliS#F=UqPI8(Qp6&mVU<>70LGSWpUw7NXf$KL-|TEKj*1NVT+a&tK6(Lf+cc)6d5~Jk$lx@(}W#$|x{879> z&g^Zw1-Aitij#*E!J?iEYc7a}MC?$4{XkKap%=oML^t&OH-!;AnGdx)EKFWGG%AYO zM#-?Z`8+X|JgH&~u(v!}KQDJ1AZ1~s1Jmm^8lZL$ zKLIHx?OYHpzWLgQ?BcjSEvmsX<;m^?fH+Ha|GPaDN5!@xYa<&Xhf(__uI|sQf-qff zMsPjqw(@jSMsLHuh|Noim5^S(oAWiqzPt}Yc3vWqu~dqGp8nH~-)>7{q)d;~V}rhV zoE2AdX0|7UwJFkJI&bG?nerT%&``v`3`wD`ad+4J6egFwQZ*MimGt@dTh+egDbCGC zywvb(W8sy&k8tCIl!^O;R4ubh?u|_Y&%InbzW%YQoUlA&RO8z7>fK;=21aK3E}Q{1 zi8%O?=ZYJ+CUW)Ju zN8!BqJa%Co1LxBpp*!K(K^BL{RWFL~gDE_EI_XbM`%d;ZmEObnnGJx^K>93;yJSa! zwfMXbxAO>uCMiND+*5%c?Y7BlGkDnx>rgZm_CZusdw+i%azMb`1^&hEubS%es$g={ zDpU(g;rpBjIZs2nj`n$`TDes6#T65z=U(ykUdDB+@_+rD6aB|>L=`l#!?-BOtM z6WB?J48Y;G2zkTMY^BvFCqgd!u`b82P%#-Os8=rKVDu@8@B``ddD5k8*is*M4zUU%xnveQK zwz-zzwId7v!?^16HM~#K8z}k)JKc~Gs(xm}^S1pru1;{E{STpc34rTyfG++Y?28Ri zmS8yE(i^&Jq&Jj!s@v`aG5)3=!;zgrV%Bm~UdeHB2y-$CbI3pBbng!`x!?Gq zrQ&7&wq1cqIL`%P{Zl#-_*9Omvb-l3Tdgra9DONBWv_?OBM`q|o7F7vG+VT5A>R3m zW!sBmQLOUW3RLQruPBLwRHhAeIp)(_IaiTqU)wt3n<&e}u>xa@i82AtsuocR(QXL4 zNgj8X?tO>WNvL@>8m{csp1riHWQjQ*3UlyYwi8j;vxgFb&EkTOn!abhL+A}AJWQa| zTy6L6&>>pPja)-iJ-nmY+2e2qcA32q-(8G_qZrG2^2{4|OioKLrAm-z^>ykP$+DyB zcinbau&_q6MeE(}8Cmg)ebVO=KZQT$#S!oDhNg0XEFtpvC_Yx`7h05Fp51Ptn zSH?H&%Dax>)+qn+2{fU(fNb>9rqt-)n5CMIb6M)UaQHi|Zrpxr(&Imc?5OVyvU_o97F50bw|nbnnL z<={f%x0!$gwCIgVFlG*w7e8S7U@g1MxvuGl;(iisB9B>FLG=2zGm?e}p@SV3hQl zz^n5Ua`&`ooZ_30)A!g`bz0mF9i_KkN{Cg@u*qv%`FUce;HuPRZc6p}xK!zb=C3M| zVK3jm*RZN^_|imk;w(W>o=GDS{pamxZbjVoRe1Q9H&r%$H8IOx*QfXKs_A*4B*bQN zqj?vO^X#||Y~-`uBfq-EA{jf-CnZ$AsW@NDZuPsH8o>;_!dBz@n#-0cuXe*oUR9Fs z%531diu=E;nq4%$_QG@u+0v=>5&joFcZus>ZON`=zE098Gt`%cA?qYd(*TKc9 zb2(h3)wcy~mAy0Mvuu99s8a&FC7@|}Ux4q2wW$=-w!ep)DH*0v7!C_Zvx#$un$MNI z!B7nUGSSAb*D6`jDK<~18J1F!Rj(y**!?IpS)D>k-jBpb_Jm!U=0o@FEq=7z@;_4s zDx`U6UZ-=9+5s>~`?vFq-|1HGnS&o6Z2yIqve4mtCH+RNyNO!hRnx22c`Mgh2I-$! zzr|fXFXCjQd{S2=Hexkonp&|N+pa);qoyhE^#dix*7hM5E+V);+f&XT9N*YuH*te3 zvY9d>)Xct`&-(K9leMU6n+Is|QEWp-woj6d`;7TqPK1G%5vJzwSZ8rh#=StQ-2f^` z^hSE<&n;HcZkp9?GT;t6IvFcR-D_VBHd@Htz@*w@R@7|G+*0`Zjz@7hcc(ut85Ua0 zWH<`<*%Q5$v9=Xtc;_Q2{4I`E)0bhv`HvJt2R9cc^?;oA-mi>6c+Q^psyv0IsN%ys zT`M$t)Ye=n6Z;R~<3PT7<71__Y31fnmrbDR#-n*+ir%2JgHkRxhch8uay7mh<7e9u9je` zU;+?E^Bq|hzmG2+WpkZD?)Z9)$(+4x$Mq>SRNZp_r~DSe_{-U8eC^N=quL=hIN`H& zwY{;GV#l|Ka$JLI&M{5Ar!S~GMKb8r8g53g`g%jwlTlj!f5`KrSw<;+`U~&Uf2drI z;dYLMM|=!8{-wfwe4Wf|@}btg`=IF%O3ATyVja`euu;5V?ZeKLgSvq<#tT9*xKr+ zDWfM7nJ~o+GNW|H z(I=&|KJLi5Ft5QXy!VlzH0}764gbUH| z$DgYcPF|)8z&r6{=uw2=PzJ|U@^`~9NCw`nQv^i#rFn2G&DMY)$snDuYj(8%~V5y2_kcu$~p z`eYevK3=eh0IH3|(Y|vchw{tivxhd~F8ch!LF1#E-U#cPfcS@P{Q+bMn<(qKu<#{+ zs1V;EK(Qvx>%~$y=0vvRV+av6V6S^2LIn8&k-x9*eKfkKn#LFv)NjXqHe9@s=c;pI z?I%FM3*4wnjII_)M84YMCyZU07s*mR+f?g8$DMpZCPZ;X`aEekTyyRm`sXaWGd3Zq ztGvzLcE)4>%~+lF)5UhKI_oz|S!}h+eJ8z}xlgKeN5+)lEf9ULb*lc0`_!{-=LL#? zsVaf;nd6H6o~$%@kGAVjjvnevH^yC!0wllGlxc^tj#SqDbbmUvgIW?N)aY$g4 z(0IhYAUkX7&z6@5<@e`eB~c5qgyWM*S#rw*Elgq1gzM?H(!|11DZw;NVBBroe4~`0 zyFJ-`s}0x2Y984Lmw0H^XMZ5DU;pUdHmsd+EphjCqvuSsQoY#kHR1!&D>`L`y^g;d z$~tm>AGbz|$#oE8v>D|hN7z6%FnqrMOY6> z=abaGfbXd%Y;n8%bIz5Gcbmi(C13~0iFbAT!TDvBDMyoau|PTbxc^m&=ZfF_J@oT8 znIUcINj2Y7$>$hN;HO6%JCoBd5n@fX%P$e$Rcd`E2U0*5ZL)I6)Bu#g`)|8JSqk6$ zqX@g-X|S(!iJoCWJ&g9cCf==C@rs;{d9#?t>vr1~nn7)bJ3?P(6FooWj_e*0<7JJW zNN`Eh%{Y5?Ln9*%GAl|e#qI>qmm$QhyX&uOU`*^U3V_Y>H)#p!CfN)f`uza?W0`Cu zVdi7{8_XD|l%e9@1n2@bd+e)MN zurD1%WUojqKKWX>NzP9FKcM|+{HfUQ%E_`<)TioQl8#b}2f5 zw6#a**y~zXTsf5<_gOcRk4%|L1jIXtiDwO&K2XmtbipObjfjGVf=H^~8L66^R=S{b zHH0^qY^&^tu?V|lsan~NiT)`i6sri;j@$IEv(a4k{i)z$PIj{<;*o(HT_^U!9{x>@jG#nBMd zL7CB3!ArCXazvl@iCV+uR5AjcH17h1hBG@UB-4eE@y-2)%JW8UmrkBRmqRHAdCHUP zWmuc3nvMEae2N8gq(7b>O^W(-7mF@Tii9VQ;BR>p2@}#C#K6gP&H0{A)?s0X_2zx_ zGfqX&LVVqGl=}X}Rv)&uZ(4vT**#}17u{oy$DUO`5{R7`SsZ}6^GsVuv~FL6^RJx- z5)(GB$GSBFRXt{;p}M@Bb_xH4%Tra)xj@20bK7=pCVL)(MFXI}^0DPQiQ3X3m6&zE zt^&Q0eL0@g$IHW-#^}I|Aa4Zl+=qbaIEoynYq)v#Q^V`@N6Z6NRVCP4?R%TGI(?^Y z&bj9*RddD)6Pq;vVV&2qo|h^*kri>sjvd1?M`zx&c;EcUX^H1lf;B7n<9olY2;g}a zaj{P~e4Hj%ZS3WctapVvs?MdS7RHk8rf<9S85BF>Yu5U;w@PaZCmV=kjy>idvyVaA z<32$rA!Nw4gEg2}dU1hw*kptE+u175nNs4HI#pq8aVmnPIp1|d|Hp%?jUMQur1#1J zU4F7{9US?XKF@oL?PI1-N*aeN3#V2IrCLrL5LnEdNmhLfL~)e{^hjxD`eA{ZG z^Z30Cs$<9WY4q2G;^FqBka#a$fK^jx_?wWwIUachr28S*uVSi4!H9Q5>H?)yUPn9i z*27n)TvkF>Bf28wwMxWAIFGikA?k>95|HJIA$6nd;iYgSr6Tp?`a-#PiHuc)-D^e~ zwXrU-6GICPCV)uJyDBluVRZA^PR|-H0G^(z>!~VcthEeIhl`{LFkGN) z(JvOf7#l&&o6i=E`CyYYwOKlGP&H8~8^2hzXW~lT1s#SRIZ}XaFpdQUqLa|y*z09% z4y#eqtv)($+1Se7Bufn7_ZpUn?Hf=q>OMFpWr8u|6J+xllpD!mgOSaKsFbPYB8!RG zDQv>(_qSNBC%-38tGlr=!|prPTI0%09QnPilM-0HLJPR&MmP^aujkf3`cQzQJW4Dq zc&7Wxr1~U=Ipvp5*}hf#P*|bpBB^Q2E``L+Sp4m#83a=E4S%t`G~w<%G59BJzi9uS zu_^4}OR`-~Q37FNHL{25W48aVsin^}J6G z8{w!!26U_d?SjF9z)#k!xSgG>aN_U}8l56I_(WH?$^N!*g;iL2adx+5iq0+?$QN!i z{kxqBgxWIPpO?dFy7%5Os8gLN8xk$m0R4em^wW+2=L&So50Pevca-JtxQA=ekgrD+i6rM?$BC2moKI&^^9<`yU7aVC%38DWJhIjvWyz1FY zBCZOt_Q!qM`edj}L0ob3^=Cff*XoFegy<67XxO|}UCle)_D1jWq@!rD+;io_>PG`q zv!4Jv0X_B*my|$}Lvvi}0e9-+Rg8OS-ELDKc3mv|eraVO1VzmYd(Q##{6E|6__oDL1la0)aWxRifRZfg zc&*?;vJ{_rrH=wRKdhednNhDu{qn}UGGH;FRH@^+Q0w7TV6%eTe8dfH%HBH{luM*` z@8{BE0x|#u^?mf;m84||IBv(m%Hin36-{Vz)loId=$t;JdfA{{YmZfpk!wsK`v*X*NuR9 z{A#RbQtQN<8*h(3uqx>NY7c%wW|?BJyGAmIzYJ}HExXz0jy{z`xG{5z*-m4kA-ZP^ z?RPm`wdZh|9Wmzh?B3=CPm384e4D`hlMV~>hrSzDj{M$!0yplLu5LO#) zIn67bzEGWQi7-?h3fpZJCfgm`tG$5vNdB3kl~Nyk4>({HPN}qWJV7cEgKAoS3!RA; z8?KOw`9On3^T5;6uYzYB&THioX@J)71E2Z%w_1g`N58wmHn5^`HXDaqXoQ_SCv{Sm z9OP!~M6qNq)SSmHYzyCTu>fAXjobOcOK0iA&efu|>oVIM5XLOQtCfiFKExv90KM)) zC^z2jm_BAForO%OS~cKC?|j_Q{aUw~Z+93*nfFkbA>yX5ccQEa-$c>qJt%4=n#t7N z|G#7>L5MfP7Ot1-SL2P5Ky(!|dY2*u&kDp2#*@^agriHJasb#t2jJG9_Pj@fV`I&Ns>wUJRa!>>D;?V=*(V;PpZ z*;KZg+6^<4?6@(j1HkMapM0h5C~IU;f8xlRM5##GYt?R=c!TG@t_{G)cK$73T1iR^ zW}jXIN?K<%u+ZZl>rxBl__f#>=-PMViiH3lE4)E9u{CFn5fMcNR{kPDXX(x@z)>3N z0&`X9L$b!ro@NLE)=N@UhT~eJ`60Grf5tN9La~#=2h^7XIRdSC(W?2g0Hg%AK$@fe zQ5U#poK3b_W@mgAW#&Cn5S79*tB8G#DljS4^`>aSM4YkE#y9m zTq*)rJ1%3wWZX?TD=07~1ou$*)b!y;w*0xI#>KgFw{7vA1q3{ZCnt(bm&u-1_|G;X ztVyM*<-eXpMc{I84*VI=8C!(f_>3I;ZUmc~_1|ge?O0$VU^|F>`NbbNbz^%?{8taobMAz6b=7#eT zyNItsz44-l3_VHx{uiqVyhEkwxN=RJBjDrpGt-J@BY+}ANSI8}w<+-kys^Pv8CY|1IL*}FV&>3G+OeR4&<8UWwnc7O}PI>#FT04rp**2`=ImM>N! zsVe7R*w5nXQzfl(grH_N#*bQoXiV??I(Iae9oapzDbBY_L(?UUySFepU#B9lV415P zvmKZehJMgJ8-;lHt=}OK@shz4zd8hB+;>4t)o?R&gZ94z&wSX8lA3NA`BE2xFiNut z&p`JWPPZqh@FulI*(EQ8>M0azXJMr5Z+e_doTBl=>&XG7D`lggK^#_((x!~`g z%e}`Kt&P#|0gA5Wvxa)+>VWQ=1)DZ~#ipEes)i(y_rr~+08Kyx~`-p!S za1ljVrwrFu1A+^?!`^baK#z)SaWVB~e5<~${(7%%#w7MoWCH(P)o8fTXGY;IcDYYJ zJ$#Ab6C6b`doxpFDT>uYwH(qH_Iyv}XzuqOX4***JT`Zt`qEU5eEk@tPC=8xs&^(P zUH;s({!Qg&^n)Nk&4taK7au1w*@d3{A7`|jaw7vkFkSEoF|nd6a&k;S&qA#|FLqe-ELuf4%+b2AQ|tLp3kRF9tiO5l14C9DJQh4*}KcP#BYwU)-14y|FitFr$RK5-G6G2IUXe@Hu;Z zoK&sC>>Q5ZEJn?4I?Hes&$U46%Nlpbl95}6J;X$ex7p9H5_}kCpEbYJY?)FgvyAm^ zdv1(8ZC&vaHNFxlg@mts3S{j8lSZAC_MOTGCtb-8iwh=>YYH+Fze`dxrsWyPq z|30`F;XgGkg^a!s?Xkdg)`^{z84{XSvuyP0Q53|INx1xH?+wZ+svoLWsJQHr0 zly_A-Jlh-J=iNV|@Lid5UGgI|_T=*IY;Z+(M!kIMS@^bn8vQ|3Y>4h`jFjL?{@10% zv+-1#MxFYR80w6@i&{lySD3eZ?{!)2xIrjQp9tz)rA7M4l zx}ypb&UYSJesugjo2oA-@6KfF$o2H8;5O*9dxSP3jh9dVrnB6DK&6b7&rPO8#)*h> zCL|kN?uMq4c6nNMsz8(AcJV{UTK0#vX0({(TQe5==mslDAGBCdcx^8KVoG&~(aFoC z9(pz-Bs|Q`G4n|GpzyYA%81P8TiK6SfLniWuxPdETYSe2>1kaQsDrP~$yoU)L(Nn# zV(bSS+#4fb4E;MXZqS2hZATwI|OSudl~We8T6CRQSK&!(lGHu#9}g=LfzpTlD(M zs|mY{D(e|{1X;Vje}(p*Vnim@+>ui-!sJ}x>D|7C(hcbUqwdY)p=|s9aV1egWy=z6 zC=$_-b(F0T*>{pG5n{44Q-n)Gn`9T+_hsx$_T4D^I$6gy#yVr2`5mMC>bmdyd7kh8 zzkl&^p69WCj`#6i5a>kgywBPa5}%{xBc0=Y;imc#Zv<&96{?wF*>a@?$r9|@cNb>I z?Qef?jMCcN`&lGoeeX!NbJdaWTMMi^qY6gkv*)QI6-39E0&!Sk7ag(*@n3cHWKT1sntuYf?2H3~v6W z2ZnIR-<)LO%Rl@2QTvf(Ko9)Yg4UCiJ!NS1+I0E5`o2Lp_ow>|4-_Y3wG3KM416oa z27VfQ`2j}HCmq4#Oy_pHgEY)ei-Gk|@`x%g5|AYhBLyvWG`WcKx~k4429%+s`3u1m zxbAJx5+5vXde@xwT?^N!P0&dV*CdTiTOORiZT&}R=wvcQpTy(db1}NT>BitM7h%_(*MvXyt3Rz>l!xNCmMkQEW5&(Rug@@_ zpwu>cnf|$?p+G0E<9tkXy^3a~rWH^V=T@FIpDSmaK2ytm3K1Xa3Duq9O8A+eh3;Rw z;k2=5gJc~Suk*N)AP_lREA7^H;Ezv+Np8fJ>4WtVuhOzRczV-xuR8JmguT-4P~+ay zNEat|IQrwpmvJa&zrb74(`RbpJ_owd^{BxEkMj;*Ymy@aQ?A9sO-r_*yd`VResema z4p$BG&cMZXE;2w`U=sQB-`2z$@?`jUV$dq*FPjow=I={#ZHcEOA4uN|{BmBVpZXYw zL%zP(h$pZosdcNBrEmzYeeiA23Lq013jyRE`d=jzI60Yu%JADC@JIA;;a0w77qY+5 z+cHx>rUkaIlE<=qtg@aN>~hIwH+o}M=s|5>9W~hLSot_+vpN&SFdiiWjluC(`;~kI z!h-9;J1W^%f<%fWjoVe83-;9JR%4!~9M}_$hqqIQ zb)Bb|#lz;ryq| zjIURXX6I+Bp#SmK?De8Ym>8AVI5|K0$a2GJDaO0@YeZ`aow!;%KM(Rd_xh)`^h1+$ zO@rpD^jy}<2vXgo<)Ha&6f-}~YFAt6B!;6US#ILKr_Ya*m?(+_#&-J`2kX_|R<`jG zt42Ifi5rwSc-&zR6d8*LT>cFrI5Z(*=BK>20>?#!8vuO3|6lzIw4kW$V^)4wDe~E) z(18hjmGeMbcvcmkvpcKO8M90t-#|~$T9*eUVU(41E;b7*yAe#16MowA7;O# zeuNbHBSW=KWmUE&w}mbw*9-Icd|^?oYW8cJ?%o;Q)ua(|F^*GA(YeBquYsuaL>Bt$ zb7ABwHi{P|;{rRQjpT#P2FsCRKQ?a%_4LHf-`gi0-K5Oka?zv)MZTo&hW9&wnp;cC zA``(8Fmqkr1Lk5`mSmdP{?N?sCC=~Dz$rR^_Z$N%A$~?J1^Z6icrN)-pSw93ln=#* zlA~{kVg)DP^fwy?>9QAfNLPaAlP`>ZTllz%iq!fclEqJ=hqL<5M{m@mb2fUDhE4X% ziur%LJ{S-ey^#YLeEdhBf^+e#U60KarJOO5{i|%uI&hckoPy*Jntzs@E%w=vO#CmZ z`cSga`6Z@caPIZV_Wyodk-^nWghEzkQpDDLXdbAG*&#qd1^~Wb7{3Me_yjo4x8;1iy~g>iotuY<2?!+>dqy?r<;w%aIq85n?rXS8KS^RU zH(8nrFg^S0Ga&+CfglV5RfN@B=9cL0x8;ZZEJzf!kN>k$kzI1+DsSc`9~4ZOE^_x~ zW;ABL=&uSJK)0CQK?&M3OK+4b3QR_MIX@ZT7PzIcvj37@*4Qz^&&`nr_o`tcl2L@P zvs%QRR3vND_hg$p#f=8#!+H!WdkNP217$s$Q#k*6T~8|wPQrXEdvfU|x%ql_cv&{q zlfmmX+NA^8zz56zg4COq4$USpwER~$I-~zNVgs@aWshSwO0X?YHsAcf(g-;lG^H|i zL|RZ-|G{6~Xg;Hz?z~3<)p98m=Dtgx8(8Z%lGsib^@^;8a`dkncIg^Lt>j;iCe4Qg ztd1p`&{@@c^Hqyg^LAXUO01qQ;{HjCvVrHq1FG9y@j~#9C6A|FG8C?Ec3yXKhupNx z`Uy!@%YixYsDv(eYYy=0iyhf=q-JKKL(=F{w6DL*^Ra!^P}}381J?gSG_N~_gFib& zPYyk*+$_y<3IG-bnidpe)uq)J7~;4hecmHRs|?qtF9YNTksgS$%SAyi!&I)`7}3LK zy03}keU-WXv3&`wDyyBIi)AO{GbXd3GvUlHcPl4|=yT8g?4nwm4PJ1GT3pYp6vo{uPGD|Aac>z!2H z*DNr6vjMYPeU_3Ev`XLcfe>ceyE%nU27b2jM%6YOvGmFf*-FEKzN@!mB6y@~cB?s9yX z2oar2V;6{RxK8viA6wcg2KSC;$&})undHn@q?|8Xb(+CDUs{)y&+nith1X9ltT0Th`?O{o+%0gq{s!VQ)LOn##4J>u)0<*6h#sP*` zK3m?ijX9r~#Z+H}7oV&VuCmel8A3(+tXWQLv%hxqAygk2ClGBE^`b0q84WejCGR9W z$ZyKyy5@BMc)~uu-ghu}DdTiK?cK+-CDoR@e)DR}u@;HVbPgE_gIfT?9--}XtNA&U zZgE%KfcdRrM(EI&is>f^`ducR&GVIYcS8s7q+gl_b?<6GvbFa_?-+16hvUSET4=E z1mARyXl}LMt&ZJ0N%6U*im1JZGtntGLy+KXv4T0ZVaI`PW4+~Y@KkXDAB04ae+Cb* zE&{zEK0qo%)Q-U(>olja@Q5cwCYwWf!}sx*UB+v}f>y>Vh%vd53%w~Rojn&+DS?fV zp$s7Z0QwzZov;Cwl)e3Q-vOYTNGEVtg>*YmfmMC*VG|EA{Dxe0btX=63n~swd6L%~ zAa40LXnrZYin?%BdxDF^$#7z)E=p^g=j3{sMe4n>@slIR93lw>s121fr)GK@6MijE z&@KDWb|TyUM)O_y)arEeh!MBa(FA|ram$Xuc{qI-vgHB| zd;xauZ34_`Xz)yBsb7ieLD*3%-Jp~Ymh236O)X!eYsxlk(6nnM0!B?t#zaafARS->NVI7mIt4KHv^U}tAvnn{VVLH z3ZoP~7dEoyofri&e)KV!nM9HY7abX zDQ!N_q>P5O%CpLR1H{3_9R^vUA{!g>t>G+R^OJ4S$gHnLUOFPL)3R0c7hH=A+Ko=2 z3?X@80+Vq*qv4G8lIu{C7@l4?JIB+>=ra*LBL=<_2 zmNCR-HC4WFt&0Qjm65Iuwbgf6_)b6}Du7KEcChZASy$xs3E1ZR-bbsJzngf`2$ihI znb)24K0;tFL-r5XdQ~#98thP{m+l6xPQ8jWc+SW3QkTBNpzeaJxy-X^eyV5t{5W&f zzgq7FI&J>aLSl+qcmx!fK+p=myoW_k8~uf*`g?2+ExTV%Gdx_Fys~HXX-y?=(1qKV z2zTW67un0e=ruS-&bjc|ELHN$gF6p)cTsSiPOvcpP5Pg0AH}CyIgWMedrh zfAoM_9{kninQva@WRWt#;f(iX;Rxa%8JH7uHXpInafVN|#3W4ns_=&<98w+X`?LrL z4oPRW9!wyp&zrWAX&$1M_T)hcwhluG>N;444-bgPfmLAQdBK^Gbz%-*KWI&@gUM;3 zHf;>Am7CrDxgd-cIuhS;kXLI?*C$wdoQv&UIqk8;#T;Q#fZOk6vr8sVLDq%1S+22= zi8Mn0zvL+u=*)-JKDd>fI|f&w*8q?TXe+yBJq{R{!2hGIOt;Q+Ym(9jR$SM#m1oO0 zv_+uRSBm^$g_uPd7n~5_m;(m1II)t0N4%ld*~}?m_^AZG0;ef&qf=!aUjZd^8eTM& zjF1?B71s-ko>bIp%jN#jVu~weAlVbF9&fB-9-2oCED?CV1$hk``u*4X_%1c zC0{4%lEt&T!Mi_M8B%raq`bYQU$ee0jqTrgZ28cW{$QirVf&M%LwmVByLb-n=8B2U zrsA;UoTT7u{k5^~b8KtgEwMs1<0gAXIL{EXuZyLVae3}>)#i9tV7Ub6jci%TBf*}U z@t5XwCGRckRG~P7r!Nyth7M>65Ht_}68y-^-T6MyR+Tk7sSeqJgA~?kT|;)3>P;QL z$#Z~X^xJ0@ySUxwx2ueF%LW7p-qD~`D+^w}UP{sRsWh`sl?b9E%vO*vf?Q5@|P& zb_ThWE}Nk|gVi=i)Jh~Ddq-wVD>jRdB@FDol|pSxw_xXKqlYh6sbIpWI**cD3&mEd z@a6ivwvTk-?($ok$BwaB<|})~zM~`6xlpNDu|kZp#Dvv|YPowckwPx{#ZNS^k!hrZ z#6`ZUwY$Dw3T|0uhC(8)Z)B4R?EmbzStehvqP;f2#qGezSE5(P{b7iRK!~VBl=Mv= zDQJndSe2JQ*{pM%NLkv^x^m{)b*B0s4Jy~^IJFzBbqd33C-)TAfs~P=ZJdNto$dfScd<&glG~{-S~0@h1p$dIPX}<&vv^%|s5PPLLGtyxQT37L$O(6`dI?)ZLlh{3H1*&1MUAxwaJUSHT-vAiF4U!-PRlF~sEoE>>FyVwUgWP2R zmYEe`zg2;P3i`sP{eRGp(m#gdI3JSCKi;!NCfwTq z!+<_$E~JnslsABtyxhWv;i-KWIXRwOxaCyRe%^lNp}!MWrxZSzBjWl@Wlnjn;f#Xd zda}IBzP|a_z^<#+Ec7^fWrqKal4RvZk4i^C6E}BDjN>CGcT%2jJ`@mjvp$6^NC6I6 zS3wJA%G2aoT&%iSPSDE3B8J)E^xGI%4P{UD;k#Kwq(P%?x?iH>U_vNJWnTVNghSs4_ zQhH!#`X+nXHXS%ikzczyz-O8$_M^+SpEeCxE8M;PCcW`(1e>VC^$gH-sh&yz>3tX{ zc4{*C)VTZo*+(YVJ6QrZm!HSfj?>pE-l_1QxWQ5%(#=Rq5!hmgrn0D=Dd!J3=1Z)p z`oLn#7cRS-ZUWz#jk3ZUk}d2;#+BXra18JA%6!wtp08HLCeRRReDDjf^}dm`CkLf@ z`oK=kNW5zKczHh74;;ug@)#G^0W)o*kJ7I7ZF}yT6i`uII(+le!`mk*$_Ufy7NsY) zVfwv#kDFWHMu-LKxJB78g1_mv?>HULp+kGuGqROOZ&eU)H2CR}u}?(h1vU3J-vY>T zw(kcEsRn2%|2YR=VK3@&1>4rvYXQmC9%R zVHV19FOVW`0~F+vVJwZt_bE_hlbLbmkV`_97FoMNCB?f@m2MqmTEsJ-J15KiVtrPN zpY8fTaG9>H)TGOHwI2s^q}KTr%_x};?kkLMJl_w)CJ<#c@=xFf)}^wU$H7z!J7{;O zV&Ee;bN11WozY{=eGC&nglw!q5%#*{Z7ADihvoET5!4k#H29PB2Qh)G%kN@Nps0?B zZ^YH^#&SOw2%0#XmlvwTQNKayap$y9jh5MF^ynv048Axo^;hXniugNZT&#mNgnaef z4=owq>L9E{RECAOH*WjQu&%r8$1|?aOL0Xkk-E0*xqPFZ(TI&C?g17(bciP2?K`8q=ih zrN_;UZ$rhbbwJJk%~Om#xT{O(CL{b@c{^E-FDK#LB)?WJ17-xSNUy$VB%EsIr+$6N zkGme%FuepSkXEpr%XLH=xZk~0@9}Ax^GBtYn&TXLyA#sB=>g|0zf50xq3py|ZCF{2D=5A7#l*;%# z~NDWLBT=seNnjJOE}fb^pTG2U9sC8nTyt^fZ}J?D*itBZC@(52-o~ekR_H@ z-X-|EX>lI&>kWVBHp?P;kN8j1%fCz>&&ohvW1L6Vp&R^0k02MpP-eW&aH~d5znKkjbMTvR%TnkoYe0C7e80r~=@qoxD6Do?r54yDNEMNdsX}ecymx?hJJaG{ z`tI4L_ffj#{PE@1Fs${d;)1W4LL1lD`HjFx8bLbK2%Tzes&}v$x=|ovdSM94FTSj2 z1q^F816n?hx*c_YdQtlFsVT)-U`gsuHhExVVxbkOaMD@X!x4g6ZX{>srIBWyvPE%3 z6zmi{f1&>>s`pZhm!jjYwakmO{%OR_d4o^E6ZZWVM(lEfoyO+r=zyhYwAErA7Cyf+ zlm~w(_%A%NxpL!b-nSfO-;D>O-vEK*6be)S9t)o{>Vm}B4>rHf%;_wmIeceVa0?(w z=48e_hI^P5KV;MO9VfmSwX%b&5ch*?KWo&k8b%q#e)cKoAa!k|Jblla`_tboVzGvF zu~7I%Q4IIP2HPy;dNON$9aWF*|ARFaQe@uZi1G%f0Og|w41Zjy7{jyTrd}IacY09pjpB> zTSDDlNLpW52#@-_D*K}4Cp2v&wYi4XYbH24ET^&4)uGmMd2P}2TU2c_MfUOO5D8!+ zAecwKi|(8QHU{e=gC1Z<>%R$5i#bEG{uxVVlpUV!uYI=uJTqZlpGtk-wMM<^jAQkjza;ZW-(h@B{t{!5)xzqA-313^e-wdO z!DQ{pEZB2u4pL+&@T=Sk@6eV7oF@fip4P@sR^G$DDtVD2{BblMigP6S@BHYF8wumH zH?FurGs)A8`baam@Jfa>QB26}w0a zY;q|TA>@L;-B{B{ooyTi;bb$`C;A<+4ynQr=%-$VHD#l$B=*JvoH zkU)3Z^fIKncHB)Q{iC#gw)sT)GB$M_>03>7{~_xLEh?$RxI?L@Sz<+R%fjO#Ml@{l z*sWCzCDcyPM{VfZLB-50yyv7U6Lhz3o*Is-+1j3dgods?(tkUO%XSk%X$Hgc6dY0} zdfN@WEYAYZKU#k?-@44_cZN}~COeG-c2rmc8>_rKu#p_~FkG964SnXR$%Tux^Sea} zd&hb|vpLmvt(Sd)@$8*1w!tw@7i+Fq?w7R_9DNYA43I853LTs23}?tW1;N@HEMU9l zGA4_Z56NV0#n=x6Z?{@8Y;?wKq~|{tDddQW`77m7v9?D<`76W5+!WL`=vxV6`MhW9 zL)x?7xO>^j@v`vN|5%YL@&^M*59m{k&etfu`DpXx*VJGZnfBfkQ2sps$gPp@2_`v* zw&KZJ{^GN8s;Bi_{GjSIyd#4QsS)*%J5s?H-Z#-r4<+>aNk08woB;EAH{eX-VUtCmEK#aTe5t7seo&| zBAtqxV>dX+bq01OU>&}G=%#pqyf@u<6ywv_&*M{|jn?#X@FnTBF}th_m?M(eYhX+A zD7XSIYuXcA$J?A+^unSS3UA*ioE4Fg2A;q~_e->vE|VV-xq*EZ<$=%x__eu88{MOc zMx8fXJaN$6ftohw+y_&VadO7U57UO9kq=nMY3raL3HM#NmeF?$+seFH%#k z=rvjBu1{(RR23;3N%1f=<*p{#-VVa4(4R&YTq&`-xOgPNh>F6!`n@*2?%3$`-A|tQ zPEqn2KMQuu!ypd1G`11hD@H?aRR+ePh|mW14hztVNMTd*17XWO?^)zjj`xF~hr?=AMg^{g--#Nxt{xRahz(9$K1i*8?$cSa zZSXCmD^Hp3PX7hxIOd}|PtRfv3QHFJ_Ab*zNm#C+sfpsc>K_jz9J7-s`zbb>+)@oi)(Mga}w|M7Z;A)>@u9a8ZLTE#qLAoELA| zf)*396cF~&92Dsj-IHvP3kv;tWbnvk7OxG@#}3W-++N4FY<6w_*zDgltYpcATgA(e zhuxo00Lcdy85c@xr`&$r1sGa0Z}Wk097{2>frpyw)OiIwy230k?aObC6YU%q%QAmk8#=_wci9df)4A$ zYeCPmFT4XW6LP6{w|Us;i{zD!D6UNq6v|p8%s~=KqyN1lm0cC3+*BEAQpB<^OAu@n zLXX?4NM29LCDy*^=hE9NBLze`*GRyYjd4U1%!>cN(w;VG2WTXgnv^1vN7LYVa4;EMZuX9-&z zqNsHC*ByKG?!an+MLA~P*RIS_Vyna(ebJq3Q{lrKQs~Zl8#rekn!nh!Q?w!6LFXXBAZ0zymZNoW;4S&kFs>709G zDld3Znk1E%=i6qf7So14;|x!zs%dUU)xxXMix7k91t<48Sy`y{vrI&;-%?R?X*1?5 zMW&UD9PpP*KOT_Y+z-$up~3UFg@FGbl!7qTs-}I{aE^1%WG;4YER`Wu$cQ_mIl^5Z;$IqNSQr2Ep&3UN#4U4kcDae-6o*f zJ#kM5Pt5igs5?mWP%ATJSX6lR;hcOT$MOl*5)fCTRF*U=4@U)S7>r2M*EIFIweuwb zb^rZ>_NBetGvs23W^i%09dVY;M}6IfkTr>P%$T~nSF=k!uUAex4XOSxuxuaJHbrf$ z=Fxnj@z_^(J~{1OP6oavO30ldo;Asbs+V3`21V^BbhI4DAnQfdSFbH1iev!mc?iq`ABx`q^%ib>h|1DfR)4?98*vQrf;9su!Ksob-SHG5d>;c(}u|y(BV$ir6X6O4gT6-=EFbBT3lUA|hOW?rh#Wk@x=E5Cm9_so&SovqkXi`1IgcjE0VH{+oO3hF3A;GV zBdRL8-dlR$$DWwwe$eLt6+-L|?aioG=0##qxOc$;_YGTxT8HY(%;!KRw00f8`y6x8 zj;B+8RY6U@?h{o1)Pwxi^U!o^;7S%YcB5(SqsuPPRM8c*Jo@4q zAw?A7hCaL3+#UH$NWyVt>xO&0)?JVIL==95^?4r8%>+o`svnxl2;pzv8Lb?FIGBa& zvsZfTS@Z~xtaOF!aViXQb4JEyl`Wq(GeVz*nfD&hJVHiyFwe-K0DgT}2WH;|#6I0E z<{z>^{(P6ewn*VSaE(kU8bZvn3|H&!$XK9ai$}HHt@#xRLKA;_ZIyRo z9$vM3zlF{X8kNj&gPS~YG&oLj%we})>T9Qq+enb0$g%i$t*Y_zd>i>p1B;6IS^WKP zi(KFbwkI%m$XW!Bxa3XH1)PHp$ec%D)*cS@)X7LiWKCxg**q$O%%wbJD5V)i8-3U_ zN7UL{6kF+UOOC8B+4v9sunt&NNqN26-okN*Bh-irmx@4qRO?i0r4~mR&^047SyI)5 zrB{_?iU&fGJ7*EPwLg};`{aO1xtxJ0bZrhcDRU}O7Z5v?!O9G5UG4q@P`+WEQaNMy zilT?#S`jJkG?o__2qFM9qHH?<&AIn5^P;fMhFX#KTTc3Z1nyFZ3*MSB*VvH;m9x(8 zL2UQ<>N4-N1MzV)7{Jv&psPm~u7gsW1FpmXz%H46vG@CaYBC-{?9Z0I8TmYQ@`yZ2 zk%_4?)6SU9or>@f%+QW7zdTY_)33p|c~fyg_xn>tk5d#CzDsp z01DLK2^@v{CT8UO6Wg!F6%i+9Lxb{SyE6IjdhmK+s8{xV8y=Yl436t*W8$Uwq>j61 zUM$noceYTQ34?eUQ06infSEUlaP8 zH0~D|d@3C~;2_7CseHQ%=p5?D`}DNWFMXVvzbUa(Lec=}xn&%DFz?>Y(kEOu$q$yJ zvQEiOSoYSWJEO^jtP(*nWsx_uKC+Vda6_x8``+McLWg2#iiq%or8r!BRi1r1zWTQv zLk+(?T-_HNVg4~==UVw@a7W*&u~Z+T5^JOAQsgBJPUs<|t0JOTF~m7SUk|Xm-Jift z9vm9Qr8oCtc>6B^OZN_ee+x!9ga+tbU=rm^5cgmfPzZw6ogFJr0x1X;WdI5BR?jo9V1z?Hv!iEOKM(xiNj zeHhZB(s7Fnzfsub&+dO>znXM4vqa=~panUth0I9)_3P302?{;djXl&pSN3Z%V3~ir z;3@~YO0&2J)J#CX^8Vw;v>8%0F@mfaUA$~8d&6bYQ|4_+^iurbuE4SG+SM{Jh^*pB z+X+#h;M+QfW7O^@aD(Wq{wLcbAVbC*KFJkj8XZB}XiZ25e&5+r^uDGuDO+ANp=Zk{W5^^>jgt*R zZENB+cj=lSMa_;oYTH+i*or3?r9Ej_5ty+IuTmi{a|msaCHbPG;2nBo$>#3gOFr6_ zfhh|B@i5Oo@{iB`6UK6%rI}?WFmn8Y(Yu_F_D>!w$d=?NgV|s&KUMLAe=gMVKfz5sk?0BzE) z^{=F>)t#c2@#n#hZ_$%y4%x_0vhC;Qhh@@F_zypDn}P|WYd2M1(ewwAXWl3W=3#gW zmvr(J66^9lz?p5gbJp%&?hnYujPE4lvBbD9&*NCCdNZBxRKXVvi1muZn*VmTT!tEcDqBGu4S9zTRFyF*&FG5Q+ zMCPaY#(I?+{K4;>c~4{&C-Ghe?5aI za7iYA*xHs-e4`juw$WktFjo_@x-u9%vm@U13h+clV~|JVjjhq2tD;MC@1%_MUG@MX zTpnWevw%t?&fifQZz7r-k_CzmWt~^PXv6#POYuHz~Sp@qA?j!BTZ;&C@gbFOC|DAE=;Lx_}rEt<$ zuG`6*;CkQ^ZL&xy_9Cbf{vsc}ee5WTKhjm^Jvn#i>a?Ow-Ny2XEh`$26>r#Y6+*hpzDy5*%u!4@ z{+ekxBDVK4$H}ao5-5w^^G>>#RF)6zI@mYtbbJpczTk~uOs%)wUsStsq}aa+yOxkQ zuy^08+X^2X>flj(rTM2}XqUt2ALVAZL^xV~rzW&9w>K_Gf#ZkhX;RJN1(T=rUf){c zv`H0l8!zosiG z0PKid#I7K#o3&SQWh92ammp|udGp$PYIIT%fd4Btf)tzaZ9jit?QC=7A^YV5Dd1{d z7D6%7*23i(acHpfmjBF$6MnZfukzAt9L8u+-h%(iV9E_XDzk7TO6ztfV?f{(26jS ztc_n}dsP?gT{IpT;vJ>mJ-?(sP_Df~y;p%1JL{YA9IBXLS*IQ&MepVfZ|!DFZrqug z+IYGWx9JPPS5dVn4ON$YAqGC4~C^bQKQ!>j@-Df38dQwBSq{1%4Dfd04{dTKI&p#02x)wJBHZl`zjuZ{j|>)TN$ATJd(v#L9bEaK0Qw*=}xTf*>uPt~)UZ!j}eZ@bZ_!(wHobVyet zW|-(D^8i4WC4!dBcZ~N9X$}KGLka?7O$v9uu-{Ee z?>Cx@5=Mavnj$*rZx8FZ+%G~ZxN!U%7BBlW$<%GO}oThn}vJjM^1 z)mYn8Ue!44bYY9NM7!k&&W=R%4|lS>H+=ix(GccdvHm2yaHqxVAZ%+;W@1Gn;DX`$ z$hgGK=}*L1+s?Jw@*ozW30Y91aq__A+mfrK64?W7r9Bw=n!1nv4ZgXg-LU#l=s20SKCT#1m&?8|avpJeoL zX9Ot*7b~WNR^A2gobW_5B2s`cOu=|(NJ~u6%{yX(E;OkYv>)I#jcC4tQ#^f~4y*#ASsWPw*zdA3Tu5;}c^D_Ff3~xST z90&)t9-W<@vbdTR_Y>|0EgT-q3!2?@a6|YOywksOO6}>9WTO%K7Omq9&Q zhLP~@g-+O%*@>|55rOkd=JGx6Sb|@3_2AS&e#eTohq*FyB_>m%R#QKnU^_dftPrURNMF9>XfK&EkP4X6 z)aUPr*ja5wM25urnyh2+Krda)zCY@I|7IzZEFFbpz<}rR;=DFNAZ3n9I zr8iv&zNYu4UVpvK)Hq*Z_-2}KrVm%GL^nQ6cILX#x))sq`)YHM8^u32N-Ibd`JCFSo;wqxW)J06nEG?9Ry<)i z01YO({+WCZN@kpA1WykxOZs3eByM`D)a>n&l0bN3Infh?N%6p+kdapE&vS#{eS)t# zWwzr8@nP+sDxjPkXP}5nF25*^ag9_#c4bb%@78~fVXHj$~^i6)LXTEd&eP-0}DuCybX8>TffffqDOp! z?>Xuz<>53bJ+v7-H$Z!W!cE{~h-dlAcTY&Rg66p1oS8TQIPg*!h5Wt7j<;5QcH2mB zU=h49IA4Kb#l4SeV%tH3?q+|q6zKJ9+y4@XKMVo^CxZ>O7C%!&ik`|}e<{8hj=0>E z(zKdOme+wY7#N&XVWKSUY(X}7N~etDiYxiYTz_w96}lJo-%(F&zqGwmt~3x3d~s)f z0P(QFeq3v`TbggZUf1id$9%X8+}mVL)FqGRQsSy1V^!r&-lKqYmkqdAY$`A%0I)<~oSP zaXfH*UuVOEa7y{Z))AUWXK^=@M;0jh3oI$lS5R;(apvKhy)M~{Fra>NX z)qht7istgKK^s z^!Bt_QoUPCwdDaSHB{KPbarhWWgT;3wdt;>gHFT2T>atk{JeEbPcgeKXi5Q?5czor zf1ZcDeemn)kE0+qQyIQ=h7633;z+)bd;l2=yPE+b#(?}|fec%#inZs=l^{B1xC@;| zzP&0z)&iN2Y|vL(lFuB5eZ;{!&RcT+kd&5qSw=)SH!%8Yrgr;S1NdrNjlT!C!2>fR z(g)i+6|)0^Xt$748sV+f@jLI>^|CVJN3SzlKtmg-BYd-WSEJgVYglTlVeWB=r|W@!SBfRN87)sK4D{=SJNLffxS&z-s$gb7_n zzY)IOHN{1*RMj!d74%G>@eaQS8u+pWGr_zbWjg{STTQr|zEPK>zQ`Ps-BJb&uXn9TW{`uQ{R(u62t_bpR3 ziI7UinsE_nrTYhGPYqmE3uxIsD=?=>hT69P)XwUS1`LOcpyowA(1zJsX_=?HJ{GGtB;BvQV}(2FVfXyXZ&R4IS6PeMQ%RS0)`WUO1TR+2vH=0;U}JM8!gL?j z2Hd@JPJaure=@2|Mw8`weF4iEvag_-a|7l?9;AN`PmdrSLUUGwX>P$RCB2K?+~wNM zn3mZOEu_12=>UK4U%vElee(`uVXwRgQNGW{m)e?N*pK_UUMni~g_x70#&m?8F4hn~$oei{8{1$zB3%ao;M@+Ll{L zl3>M|Bhg$1Q>H1W&7N@!yq+yJmmClfJoTCPWN-koM2HJpIBGUJ$U%FIeGQ%$F;Vy2 zzlF21Bo9iHZ8$pZ@Hw~Mesr{^VKE#Lihl!J!;3~G8a?Ce{USGRv|xxaRer4YapM=k zJ-|VK26aDt({2NsAjKYTw+p&!aTchX`%W_{A~QQ>08sq0`zNpt{upk0BhuWlKj%P% z*^%<4Be?1QZl@1quPJ#oSB&Pnb6!n~u4@RHIVHr!Himo7&uY22vWx}!UZ+vbWuEvEus4KRH@D!(WcromjwNY% zJf;|5E-wX)bMjTTQUr|yl|V+5UTIND3)Cj>H72NO`KA83pN0M|3sVkRm_l%%!aN>j z#JCZJs4+>?s&$n^M2n%^KC|s%nVR;O7RbG5d*OroCaSlUHO;Yjy$LreP2s{9VUMmq zp4Lq4@gUQ5jiI)tLa zTre*w6lgPH^^TJ&HWuFmYPxKwzfX-G$mv^CsM!lW7^qoGOcXj5M1_vB{3y^H^)i~p zWP_*vFE@_1n)DWZHrbHf$Zv`W=bdS~Jq_)IqN5sNueISh&Np>tKJS z=+Q#a0~_b7Z|wAsJ)Y)!+oKxw`b;+KewKC5_C;fD3CpVUF^uLnIINO9Paj%T1@@vB81p-0{G^ha)4*Xd>|SN zK7ycH>ABYw~luBj;4Tsc|g9AD);P>MS92%bI!PlJ2AhtQ|>k&g&(f7A?98b+M^s|D2TW z6WC;PO1b2CXI~0a@>~k0VOZ_K)QR;xxG=P7gLAn9Pdkza))$x~x{7 zbqQ6=9kiXJL(hn=Ay?y=6?wNxdn~td1YOH~o)(}Tdmo{u8ma5PHGk+`=uYR)R}@NI z^;ynNSKcTJZzJ{w$4TBpnq?>~0AVC1_~`l57J7Zz*EDhZ?@s~o#eOxlM@Ej$!8Fc^ zK~T9s47Ov*Cv2q3Bc6#mmCNfri+J~eB0b`IVNB2!jyc1hL0q3a{<{75PhwZ!aOcp^P?l62b_Yi2_G9Vn4?+; zN&W-;&aWcU^OQUU7S!UG@l53Q9jE20gj!+o6fl4E{42yAzbJ8$PTyx+2uCZW+ z3hg#Cm=J9`SG#iEuOqSxLJ~W-x;A37jce#YKxQ7Z*(7`39GjP9K2&bVgOxo>9B4Zw#m_dp(wb^}O;g1De3^vK8QemB{&a;y9nCrWWNX zW1bNB$Ie_Yf06jR=V~e7^*4J(P{UFpdv!k#9WNOf0SM1|y7FggDFa95bVa{Y+tI4Tuz_T_nb{Bg^8{5ktoSbL5;8F8(l`T}Pnprb}`rj{)!gvq{vQW2F5OWp_ZdY_oZge=_ALm1C$di;3o_hZ-D0 zQBW;8u7IQ+@@1Z+X^TAb5! zeNyr$^+6>^KYCO3P5I&?tJEG!{5ZR1*|JwPlK}DQRCm#je<vZX$z8zB``>Y~&EBkOAYf9HmN~4EBMr1d$6e7@E~YwbzwRyUp+1v& zh>s9{UUy?#&}QY_%dLPv7%i5xUnjmGqpS`>Fgn~lW8@Ce^@>vXOZ(wwkHehoj=0HJby_%96tPObE*a*oHmCE5eApYhXptFZbKHDUqTF_~%!^O8Jh|6_D+de2 zKBt`0J;ce@+8JnE2vXI!8{KNnfn(hF?oFCR( zmAQzh`49q?p@rOk&h?7*qoB+S^X{gz#!hKP6m#jpo&zk{%g3>*cLRBggtvQaF}U?P zFW~YrQ1PVH$IH^!#m4e+!S zyieQ6bGGf-#kX3oIblEa#vB8lIDJ|1^O!;CydAD+?h%w)Q8-6_XxOPksccgNx&iNe zc;ZasRwI$vqLx#xc8s#quV~6uwXu+22Y8EGe&gZ(rLap=9Jc7(&{Xrur4P4EcPwZ- zL4}P;tP|!FwXkqRA$hWMr^n0`a@&IbfM2gK-HdOfu_;5GqKPSPk{3Tk7vE){$eZV6 zKS5E`a+u$nbKVd9aLpfIE2URjj!Ws>i15FGuMLE`r^ zrM;nCCA(m|m1`R+S6ip-z>@Fe3-KKp;$Ti~xMd=z*@NPDA6AFXZkLKk{?EQJ zT%ZMQ!Azy}lPK5Tv-grM0De0j5);uX;BPv__7&rr9r0nauBbW2n@;YDlgdNatFD~B zDrhGUG}`)2?-Uwd_$o@u0=@O67gSqbwwmpD0fr!nWV`_eKO{MP0Ht8;!oo@bFBngoh7>86#qSk5JoK5)6sz z#k||pvg10PWfxMDX-G;8J_;vAcP=X2QX%EE_M*JzL4)>~6nN1JK{~d%gDbzoy66B9 zB1~fppZRj2yxC)lJ!XH8QxJbIpw)5R{O;fhqC8U=XPEcs_-U9iE z1pMru(0i|47@*ZE^p_NzHH0SO6ZY6Gb>%M!s z*QkqV@T?5AZ%;(`!{f$QM0B&)Ysn?Jdp$&fcid!ma5rTIyS6FJ&wsw|z_m9K)_z!V z_eDh)UXbhyvFe)f$^~De#h5;=7YiRD8qiG-)cPTvr89{1xk%Ib;Sbm5;NsZHK1yF% zrM(DTnOd3UyK;xB<{NXEN~I_I>E6+@FVoU(u0_*d1O3?t$J*fOZ9Yoy?Lnrkm|xYV zn21wh4&y^tC5^WNt$kY}b2$ZGFkYNi807DVc4@`OM!A4z29@bSAf3BO(tz2mr^9}-<&>8C?*O=k`JasH`;MtHt0nsHn7SQHBj#c2aghVXV&l6pGYV5h0C zxYwfR2+MZgNBRLBp7X_sUacH^m+~+YZJ=eUxMf{e6PIG~S-xKi+~+b%KR&{DKTvaO zjsB|5>Mgh+20iCRzZ;cbfxzlgoo?ZF?1dXf`?v>OLmhVR&Ua+5L-J4eYj~%c5N(a` z+jMHXM4P6jSpvAStRxa~EFScs>G~K@0-utbb`0ovUI&JPAtv$JUiFAm*`=fY; zNg$#Q@>>j<{<+kis?dw;Bfo|7>r}(9T7%1CYh`h&nmQEnhAtgCjSt)`Gkpa}G%=S3 z@{JY~y>u&`XH>Zk+(HB8(;JfpeLc_F`r~cY?wyZoiG0H826=ZL07lSxP5|T=-hJ}f zjs30xxAZn`s|*4WW7V9tM1g82%<)g{jK?<-89c+pG#A)QpsLa%36daSfzuF+7Ue39 z$CIJPe`$`}Z!-w-MCp9keObX^SNX8zr_%dtYG&i=RlKF1+WjJx3b$Rbm^gVYhN@GU zmLjsveLcvi0|%4R>)AGvjz86^+l{N1o+^5zE%qMo4F)>#O`WNXw}Ts(55z(q;r)ws zg}PCt{wSF_%tyLnU>kY?4e_97t@n@++|1+iDwkS4R+9n|%vy+j{7PhxyMP?fvF8~8 z523QoDH|+BVo%xnUarSZI^f5T8ZiN58{yXL9cO&|cUo~0QRO);ggwla=M7${LSNhXEgjhG9DZClm!~C(D z!!0XH&6D|5cWm6IIOUMa5~XYahr`KDBjF1lSsTs?kSPgCdcrPhX1DkJ<}sXnCW-}9 zAmta+6EKV8noQOvy&B4$dOa0w*<{kMQJvaO$%_S8kTt+ijFPot%Z<6^H7)A}?<5o8=ntvy>i z6{_a^wurb{Ry{1Mzh-F;xJe7DW~|Oh`sIH}OVpSB4JrKm!Iq%kJPE2ipZ+Q<{4RQb z-J^52(q47x8A|HDu=`6fK00^}dO&vmdT!F#C^LfR0pqjb29sDDsD0FgIGiZ7cx;sL zl9vqRAZ1jTE>_!2uy~kq(2`AY>k91)GxoMi9_IBa@v!RNq+s}^b^J@7dKZwzN8?{_ zh72&_1@;+~2T-8qT%wOc_=E{2if>S*tV!W59tih3dDTuUotf;fsAF~pTo$NL0Gp<& zZqBBFPi2#`9cz=y2OV_hh{wJmo^X`w?Ax9$2d;O;FGW6!Vu(7&m81qV51_CJY(R_Q z0MHova9rhf52eY3*0Ps#x63%{Lh06T(PFRYFHfi3kXx#of2^glTpF-SyZX_UM5V9V z&mk`D9CRw9bW$(6ZN^GQL(UCtYRl{+DHcPLKhUNy$UQr4!}z&u8iMDF%^Fk(^7uag z;#g%N2x!G7$17vwF;wha%vGg}Ko>7xcJDJz^{;V%7x)<^KrAW?&&geE zEd!RswZO>>DBS~)ghleCyGP&&{HK}RU-|>Qk%Jg^{;I}Rgi+vHEh#RQNd$eQ4lSR} z6dVpqVTKF0DuB!04Z6=xvM;=*UeK4MDLvUQC8X(I&5>PI9Uc&d6YSznaped3w#`p% zmVa1cF_QzQ+hc)D1@jq^QZ7qQI186TiOG@XPuFTopAL3@^z{Xr7je1_O^jwoH}8|` z1L+d04LB*RJwevm&85)xTkUt+o!X(BH#c{YTYy|Rj#tIe=bHk>W!-3JEXogb7a2cf z{^8?BUGQa?sr8=Sn$*${Ghyeh3?6hZFz4|7FoM6AfyFQWL(L-MEsm+M?PrUn22utKjU# z@@?*w=lkBN{ug^E*!2LW@avayI6p1!%MQNjJA;YazRWR8|1`ergzAikx`+-}3x;dfg zq1&w&?Y|Y%6z2P(=2CUY{pM{7cYxx$EUNe0cJ0(O_;;f1_@WAarC61{3Li3!&7%x_ zGocLAa<$;)q|?@oakF)E;b!efz@YG`{J9FpBKdVJe6P62CHQRDX{1?oi2@O5;lS6+ zeeJ9}`o1ktne5zw_9cg7JrrP(HMNw_6Wl$&078 z6IKONF5Jk(gVXT3hf(QYvS&CC4RMaQudG1OT%Wg)C}=x-3D!wf0J=+H7Rz5Z3EJj@ zZ{WUz+w;hGCUaUZ>K6^Kp6Xb2hRD<<`u1O{$<^}*A1)07WuW*j+c<%)sRU1L-&X&I zWKUH8w!onFA8e*$n?ir(FO{oQZ>}o>4Iv7|{y)`q8FzWc>Hral`8dUiv~#2%XIf%u zGpl2{40^1k@okpVKHt)`fEEP`JfEB6tcR2#rd8M8C{0*UC1fS0q{f1mdzWTo7UK85 zQ>_!UEydbca^d57fBnA6o;*Lk#NPm1Sv35>lm!#vcqqP%X`SypHi*(&AZWYnJa{*l z`Rz7W!Vf0eF)S`sr%-dOVx7C~D(D&b4bsGp{O<)Nkp986zof&aMt%Hz9L^~c4p z1i(om`EiYF+L{B-Mh@G1ld$7sO*~MfiUvAWkAME&f8wdR-+-;}qrp>q0tt2iifVh1;xL{g~v2lh`d!FV+DFN|bK;k-Tm%_$h zKq@`VK}L@D!ewDNC}^9>Wm~GPsL|1jh++@b$j+M)C6mUU(!IA&T{_hp9x{z_H=E8^ z5B6z;zt;s-f@-q?p$A@$pTPtYberbF*!V7;_Ztnh1_iRy_gpgwn_IYocDS! z8fw(A)Z?1EFFkWR;p_53+}2QZ71zt@khKaPc%P^Ih652Dddg*`<5ropmTQVPHz95R zQi{e3-&o(CyJT&a4Z}Lr<8r3v7j9SZJw6h^hCXS`^3pj1FJaj`pj@>oOUuW4H;V1WJSwrA0(u1{-sa2hLJ*!8oBbP=r@~@82 z#(p_@h%*OJCaY63_I?s7!VRhCxdM4KDVwpB^4S8C8oPEU`k7bFN8gK~%TM1ozCmTe zpTa>r+(SzRwTFqFn_tlUG{*dv+w)$5rk~cH#8hZE5Nnuf(>kT^dn6^vO&o z%;WX$aD#2*I}}VDKJBVQZa(+=ja}kj)2{Y=o#$|;%Za0IQCF}BXVv5ZIiho!&6{X@ zSBk-&XTC2QZ!XSyR1=ydJ|7_QE0c@4W#6MU-Yx~+Qwju3_UswHm)%>Uk6$sFNdC3z z*#JCf*x|lxW?c<;Y+iNz^rB~1>)`a2?FTQOBv60R^*;aDQ$s_Qdh-Mn2k^ra53G+K zG8NU13o+kXya#$9#U9yt?@XfA=jZnxH{MAo7F7I9{9TT(Yu9GgrwT0xX!WWX<@r7E zScOUCUAQgLbo2cwTzN$yFK9rWIzn~A&CyI6Tab}}^0hUAAk z67uQAL85Np`_Z5|sZ>j8Np6lPbXbMeMSOi8ZC>~e=!Izr?U&KI?l>Vj`3EDgSENOA zL3|%(OR;mOxv~krctC8^Y(KW$6}?fg=R%d4SoO;XZx3N}-9R|Uy3%J6BkrWEtNO)m z5gt8Z?JyIeVRzk5tPA_=H!MF%1GxF;+ebJ&ZR6s&45sp^qDY1kBVowo_1*Q?GFlZ2 zX9UDXc79k9=Jv9GV%v3Ma${IEV%fdgY%XGXi|t?QC7c#}{?bk%Bezdp)MJ8*gFw?> z$OR!`OL7doVh53t+_i!l$x2>V`5pV1B}Nwy!w->@WH7};ueCxK^kLU^J1^XEvu=(u z&KNo=`jHa$W&STv1(q4~r;HX?&@;|Lxjr9EL*jN-9=IZk^qtb2SF9_UvXGRkOKei@ zE23NP>F$nL?iCy8c^m>0=Hvle^O_!*5HY>cY*!erv|P-6|IfQ4iQtPAMv{JxN#bmAY!pSyI@o;8>K!$?4&op&#$KL7biw`XrC^pql8xS90rqBB1i;I# zgaS+A3W{malO|fU%95xNovY;QD}(j=Fn;eyZqLe`MYIg&7aG#1n_RVUxfJ3cl%h1= za(n@NK$v741c+4kwR#??n!YPHIdF(SACrzo3*7iTAu^&Q?A^8zavK1x=)GfvOue@V zHfza>8}es}H#ya9XTgfqBK=Pp%pt6t6utJkqVU?pO3tVrC@`at9 zYkRA9RQI8M3_V4xqrqdGx$rM1LmK+kA@@;Sugt z(DI#^WBw7#S>-+Cc39I|aSbpzcQL#8$U6Xyr-h6i51EL**m*g`IYX2WH|fn{(%H9> z<)Q$>&c+iSWZ5ZVAK$YP8fGuEq}UEl)N4 zbZG#Xft5$=Ute@w6P0mma5%OapxN4&mnV~y}gh(Hz~ z7|~tXg^p4`9!M(@kt~8OWk4ceP9rh?;U|U-8cJM+uz~vXn_?>G%)1dZ5c|+<&yQ&m z5AiMkcxQ)cV|{JyIZWv?PWzn1(xK7*if}Mi#+!-Ulb$cf;IrN6Of%=8*SnAFCc#r> zQ9nH@(!eqW%h+0Oj7iEwQ+KV*TQ#7-Le(t&9i1W~tE+##Q+VXZYNboT^I-C!(aaFt zMd?u(NC;o6&)5_}&w(2RZH%Qs;GN%a^CzIt8~^7u|F;9JXArwNP(dXj=nKq>L(G+K z>~A&a<~5uPS8;ey;9~S9b0C%FmBC_A=vi898m>cm&F)$61Q|~S^}x!aGXU>l%9uk- zd}=#d4cFOHO7V~_@=C?luxBI`VzOy}Xo2$y4ZrQrEmJu+eK7gl*42~#u(U*CP3~XQ7aE=`Q;xQOGBia>Ry$H?2#JvIoGX)0L4R1J z?!mk3WLbp4R((A%y+!%CMv(BdlHo=#)rQADzF2BI=Oe5;| z2(m*XF(jZcV#*?zx-bx5eE(R(M2vLGL;S>46sus*@kWoxl1SIch4Kehv8CQNhLEA< zy;`l>%*y?}A4nuDL!~v@sXcPdC(=RQNDO}UQJ?xxYS zO4LR4EoT^V-ce=zN)S^m520bwc5H%O>d~-T#-|nJ+NBf?QKKs>$wYn>biPxx(8het zQfxAH)hoPFFhBJc^mytN4O8k8vB&s0O}IwZ+a41^(I(#+~Qz>UE8MCriM|BjHu`4?aD4&`DA0nz6eQWy04Cc;W) z5=6K`Xzhq)qb_g63%ds)5>Fes(H8F}M_}zqOCI}NhST$U8;ynCKh!;0Ub1LSLoRy; zyuK`C+xm-}b$Rt&L%k5Q#_OBZ?2TuoX&omw-zpCN8nI^FU?7E0J=_xVJg2a1ZdOaG zZm>lMua73{6%_Nf#Apz+;v3qFp6FHag78>lezbu6SWIy_iEv=u@4q8rlW`fT*Xqb? zW@5#!%e(6b)=&3ogUU5@j(QVotNUC(5f@{Ob{ufOD;=h{s3bDdh>7Lio)tBjbUFn1 z1X1^CyC=f=rJ*@8m`P%+-Alr8uc&AH8oZN%51?r@c-vC1Yk7`bNeWZ3t-P6|IR9~J z#lev3v`{5KRexOItTnF8SjqpS3a)axoK`zo4=ug4Nz+g123=mB^#)0zmc$eN zP3aY|2{tbfdpPcAPe>8=x&GqXjp~kKv&QaCjUr|j_Smp``Aiu@q<+(ZhnQYAmzN@- zDeG~WEYZD2Xvf+GW`wOL^i7c6Sk@*h*E~R2NupJF4M8X2e~s6fJY|h%SFLwEJ0w-0 z&VOUlF5-8%OCT&UCO@4od!ZWnl^SU#lb7X>e(;s$-?`n8rd^xLzaw4GO74P(%U75? z@@sYxZ}FEjtI6P;AIonZh2RBMO&z^WHI^)+qENn2JKDt59;{%@%8LkiQnu9G z+Q<#d>LMcTv#5)buw9(Kg*t0m2;eQF32v-R&A0azt+OY=d#zCL*wX)=Jo~V-1`P)M zh{3ZBE4DHzaO4JA!H$3pm##|Jv#k~XCP4ords2|h{QpV}gqZHYf!WrKi4VOi<_1i= zH=TLl)0eTHRYNQWf5oI&RH8F;*VH5IryC6CdCkAWS`Y=bxof4XPKqk%hKUb%Ajt$W zBl^fFQxysRl0f#nh(qQBw9&G+v(Rvnt_I_#knKFRg&sd*NuK(cShbv5pT+B1!XXEk zV8lD1PMEK9!LnJ#BdW3!udafQItjrsBjy$rN~+1+mtnd z8?Qw8-szo*SKPM7PrQB0BKLgf%j=F=!u&0ID;BxAyzQbRax`_qZD?fUJ@8d8zn_@_IZji~ii+$!2b&N9l-`?SClD128E*I`@_PD~cle(^YJEH9iWDG}fuQ zX^B15HVMvdr^X^C45^1=l*+*$G)clkarTepaU~YPVsPC7r3ukJO=}Za1ws|KPns)m z?uTpAxh8)crJA@&b zqt!+Q6C$0N56WlEiG04?uB)2_v5`Z6oT?BRnbBoC597UGfSbC%d?`Vko^!ifLp8d$ ztbN5)AMy8+1sc&49}dJUy$uT=t2w7d7hRb%Zzw_(Mx-gVHeJhJqsaTSWVHY zC{^MAP{Yf&vtXP|P}*X3L>{Ss>yO#^6+8bmG)NaWibCgH6$|Co)OWac{g;I7{zZ#z zhodVg=l3;lISC)`0u9c%-LB5o5sR3Au>PWDOl{ytv{QbmRg2CER$%r&C3_Kb;%jqD#0uzK_4_~vwD$i6@lwR>*Nk$>cT6yp3+8Mhlh zoMeHD$(9RliNK$;WmXne%M9i!C zCbGOdB{~A9)zjQ>2#-X8Z)UNp&x$RE?CU3CnZi@ADP5D6zJZ;?P3+z3bE#}v>KKoY z8Ehf>C&9QV)0A!=$2Kkvc$0@1y11Yp!czq(Yls$qQ(9FYDVPll5VLsZ z7j3~rfl~j7E_N3#Z~xk99qsjYf;Qz9%Lwtd^4Y4wl=V!->c$eO)==~1FnGAcR7ZS3 zX)>}Iv4FcxJ?e0d?uVd?pOlRr)2TA&b?D8h!^!&8uYk@msF7V;`+Tt;86=)k%Yzln(!W4O&|By|PalwtO7r#q_diVEJ_7LR6?vZS5TNnBXxw`uZ(6xyp z`^B#ASIyiLGcUC;_i7p|r#|yG!bj!myk?f`l!t>3p;mI>SY_I#u5m$r{oEBo&zim1 zs2!nkKE4-o5f#5knB3@_^Hh#SMd=loDBlU0D=l1=yf}gD!!b5}PaX^9E$k@)xYM>% zHMJEHUycFaUOnsU)Apb_m*-tOZ&l@9)sQzaGZc@ys*?PkttiLdxuiIUfWkd8(H40MHM+L zWE2j#UiObf3ZV152zf7>H_9kV`%d;s8F2GVppS(quq>W_^cbj6I{${IM1@&Vk_zp| zrcq+UXuDKQS~rHeT=^Zmspmbdhot8L2V&w_J?Iq%_!92j`jF*-3C5C= zlx;s|UIaD3U1B>*6|Ium2tC*BG$JcMJzPt7yZV?Z#jeIt)1(fYn^Z4qy0>FxhcDm+ z)71R#ev|(uYxQ9dN86m4O=`nNCp8*Zk>^a@N;FB1MO7cq|M=hvGk6jqe&J1h~>g-H!B@z3#Z+ zst?0=ZF)I>+bLNnk{VU)Dq1^Mrqtcy*s#*yL2q5I5$V|^%RGV#XR z9XKmprMySJE`Lz?;CG=ojSgvJnWCpk4^T005vHIzEAoNkwx<@O3g1i=5sOK0v+lbG zS|n2scwPP^X(xnz?S${rq&OGCE%xKkk2T)2=JVb0rK2&%-5X+z+w_#FN2%;p#&d0- z&J6l=$pi4+R58etM}N2ltV6;xQuzTPva<>{`$A5iy#cxQINH!x%O-X7C5iCmqhmeV zQ%J?x2y_u8e+c~y|GP#jKO1oArQRQGF#bchPiYt{jMp+KyI1LlAv6i59sFTc@d4Gf zX#9yqNVilhN{0WxiU)_f4i2mRZtO%YSV8v_v5hn>T;(V_5Ry z!>hi~m1QqDy6^Sjd8gzw0L7tx?`oF?{f5t4568DI<$5~{;5C|7g%L1rp0br99a~aK z8Qh9(mHJV8__?;j+yI-7fU|X(BsP`GUVFt}L@0_x`x9w0-AqjbrDU35~JseKW^d!?}vcvB9nY+mzfLH>wW>fG!dyTgP! zrM0?K%D)5XZu1g!Cm=YftjehVxp^af8epC0@vYwvPAsph5yn`;<$QLczQ6qo+ZJ}c z3F3)_$>8H=5Ga>4%9w3)%0?zfEG`^glBBL`qYm~kP1vB zOiKHi48}ZN`p7dqQNM1-&OfMaF9l`R_q#3OwF<7deWE`sLEs?gF89Z`7RY4R8Jk78%g>|EqC3|GmtufonQKs<{lfJ82>&jT3|lu$mGlkTvlJKM_qfO zu)sj*^x4LG$`lnmw{#g@^cxELn=jLxe(HkxqhtiWH$iNYc;=X_-_FadD^r%F_=O`O z7V1+8Wl*EmKQXdcK+kXuz}y8wp?W{TZ$;W*yt~p@_hx5)omU$T$NmJ zabvO;vWeBEtPjrf{p~te)*6TEO$jke0`hbNC^j8-Glm6XAtWDkph>>iU^pxaY%Z(9kMHHA?MW3N0eqoVW?{E!h-&sNN*R1t$(vBPA|@uC~|eQ`IUFCM9wG$_YQ}y7 zlHcep(h4A9dgEY#Q)e2>M!eR;EQ6O|h7BuzcDCR2HjW@mO#ERqUh>-{Aee*=>KJ>0 z#A^E&+4Zm_Wj{)r?&uRK1}Furc=kHmpS{Gwv6)Of8->8gj>5E6K;QI){bFb7NCXRH zoc?+l8QHJ9N;pu4uLJ4Kg7K|W)Ww1CvL4{p{2^fcy*}QfBEgPyy>Xf{kjc`b+!1D| zF>}X^FegNb{kqj!NmF72v_@Yk{r^;HR1TdHTdrlSM`!A3nO+p-Z+jl7gl8nA$eVP- z8^AG%e2gZ&^3-&w6~keZX((DfF^A2xt9uPv(p4ZhRM&%uZQk28?ahvHd!mO1<0Tw6 zD$r--g;(JnTX6rOPGKohzo7twARF(gBnLo|gmH*}vUzYe|CF7ZwT_NxM;jr#966Uw z7+YzSIlV%R28;A)%12D;S=k%gV$_+{%)<_iUW+UAALrCR(;<5sjKu0Scj-Izc;q(* zl=HTO>Ym~5oEcb8F`D!9#ps-eL$%DA$5NjAYej1d3pJQ+@%kjNOegr%j{6Pm?{-L@Su7%fnlo*>n4CIz$K@SNZ=dqkC3XaD)}xx)g!GW zO`n3y6+E}`G5s#Y{+kK>Zxz*BWU=|}k94t?3gZ4yp<(NQc>d4Lb5bB6GGGrem-%2Q z0*vAh@ku9n79ng(pU7b1Pk(V!{=@l^w{E;sHxP?4Q zp7o&QQ?ye0GkES6HLK4WLB2{fjp){#Io{1%Dysn+Sp6%&DGZOj;s?pWkz8qYR}D$(FW5) zmtU^r;WJ`hclVm|+514e=Jf^(P-Fgf@F;!n`Xu)$NBMWxrZTb}yBViU4~vdOalf)6 zq-b;3OEK3DeHTLijkwJgz#%*(3drTUtwl6*)A{;!PnIXf(IYTtU*MC87AfJ88yuJ3 zuvB?pb^z=iQfCbEB&Ld2dg53k559Bu1QC2{AM)Jod!ypNOSYnJJPRN5C_9;w6_l@t zDPJ=QJikMp?`Yl1ln5Dal3Y;~jY~EAL|rLW_x4WqB;n`gDnNHujFYOt5wHxVr7m0W z1|Re*qgWKwQ;aH_*sL0!%D8D7`~XRI5H+Fo1*%farswy_ifPLmvhbu-Q^adg+0W+ z#X4zz~@cbJZ^;-1Rl1L3As%K3gZR-!I_TRz^@+qhYo`oA^8LdMVQOjID0lf2C zk*IR&8IHHX6479NEyQdeQeUf3lc%t&>cn$>D=Xx(wGb{FVx!f#satE_WYB&cJKY|E zk7AzmvhgCGo-3P>lrD;Uh!R8GxK@DDWk2>%YdF&I#G*#e3NXqT{~a?x0k(^g_mx(k zZX#v6w%%C6w>I*ubgSfgrdXXo_buT>=uwssrZMZJS%$^!)`g^OmnVN-=L(hVt~wF= zVlP4?V%Zte_50|CXJLfHdbRQgd0n1XttcRhN-JJx-;un*!s~>dxW?xn2=-K7ezzx( zv7@N07r~mMguY|Lj#x>gv>xhjnJpmITPlWH`B^ASO=sEhR(!Y;E`#yn__C9AF4T_qi29LXzfiNrPKDwNzWspU7OhxTr@a2u3+Yh-Eq_)oy| zW;qvAoMR71UwnG~{-|(J8V+y>N#AWEUD?5q!6-Vk$wXfzWTYzOBbSRY__$t&ozAnL zyn5gQg*}^Pp0{j2((HSiOajD;+Sc+_bUV*lo&u2h+TS9x3)`6?B6q!SmsAnbE^PEx zwS4G(;{E8W0>fkVq>+G#<=C%FdXXrPfa+w~J&+^ala4XZ(MRLEOYiLO;+B$gX{=Id zDyx+>9X?xKs+=P2J)D$4GDua?oSam(du)FHMuP;jH+N1x#HPAb&O2nOhJMuTN+ftM z*p6)dZ;I)^@r3euaN~opWiIvnBc6Mt-d&yA5FkvAl`rc{vR7GAb`g4<@CV9;Ba&|_ zWBJS7l@mPf)eh)AEau|Y&MR|MMEF9Clz03pK1#jbm{}$(gfn-v(Yk(b9`kfW@2=6 z_#>DTl#KIWnH&m{!S|M**v{6?xQ{$hdns|0Rsh!6v7kjZC+ifo-!MVGUq?`W9T4Ou2r^s_ ztEfP3?(&6$057Q^Hc;A|d3MK0gv68hZxKn~9^s~5Nr&FokN#lr&CmPoks#(9M2}d5!EVOL!a{oOIa?C{Xuk#!voj~>AOIH%K1k#e`PIGyp6#!s3q=|`F*1V5a3J-m$rU#aP{f8h^23Dxv2}p_;Igv zZ9~BPt8+k42-F8K%FdNJF*|~8gJ97IA>Tj#64bvZkG+us9K;V`P7ic3H8!8(Kg{`` z(V~ZLkonPbQg*!NMBM)w7O|4h_96nWa`athV7{n(Ubwn{ad6i7p4Ax zw&kB}OAEsfL0_L?E)X`lR;fS_1zmU`n?+qA8#wi?I;%&MrF3<$h|Q|uJ}_9*JX)M z;mb8EB2PK!ArdEz6T0!mP8x>`m|n`0srx)Om^H7T_&5-~^T0bmH3;8BhfxYWds3VS z;(9Bf<<{hsCt>h^hMI-r9fm8{e6l$?+0An3Vo02_|dhA>SVFTNt>=@V)cf;`?#e<^InP zD^RH!b}FN=Y2SOPs<)zBHJ|o&;&geA`?W_xV=8lXmz=c-Swj*v6phLkh|g-rR~KKV zdRb|fEWxeGC1bEW6)mr}IUjiStoA_x`adD(MIfRW>O5fBzp>eU-!?Yjh?R>Lj?(1O zeYSNo>#Y{Jd_UbpprfTW)@H1p5ujB{r{Sm@wmLZCjZ-dk(1KHpDIE~YhBZQdU8Ak0 zt@RQ1?1m!A$hD&fbdo+Sg!0ULSUuVJ4A;^rr4OwiV*buNWwZOXd(K3ns6j1%shtCk zUNefiYqeF)%KCxsL*3V4o)Fk#5Cm|(QKcK6anl4CM!96LGDjIxS zaoNJR{OkZrUZ3sMUZ~7*O9@+;Awp>bR^f>7i%4z!Y;=VE9eI>blcL5e;?RDr)ZOLKnd|^m zB7AT3LJiw=L1U4Objz3MB4kSBz=x+{Sum-dioF#p5M*C%5*X=WgI_USIOPG3sv&~) zq75ffpr>k6@Ym;>BDBg?O1=9i$fcsEc7s_^uOYsHJo}Q@Cq8aW`(_OM#)30W-Q$NE z6oT%I%~E3i{AWbyBVT7AATz$*jrg`Vv2@LiSv5GHtl_@4{cYf$1I7X}tq}iK%AJK~ zft|5R>8@F;1|8NN52Xho9Mv4^|2mN1UmJL5n0H&L_TrzX8lnD=S-&x{*JnNOPBLN;&JT|~OI_^Kw8n%{e} zk}j-zWh@O6u0@|85Vg}u$)_87ZwI4n!C)0v#|n3kHcPv*&BUQZl@sGJumnN4!e7}I?}acWZ?9WR+-P2Qd{3xowOf zbfy>P3Z!5J)^9egIxsl6QPgL(d3W#9V7k(su)sa&BG63Cb63k!xOtQf%xEmi$M5QZ zITRA2^b8xS?xhpmKxy3?(7UyBzMprk`(5kp zUF-ROyZOWi78hK<^E#*FIFFMk`s*W#PD5f(1{>ejZSoi9S<0BUE|=EK0Ik@S6CQcd#TEgTrB_O93Iwp{(gNTa zjE6pF(@RGv|EjTzgXz-1^6H8qex6aX6I&1A1!RPSdnVQPAX)@ zM$8~cUzUfqjr`Pr8M&sE#E1%hClw%F2oPx|8`9~dLkwn*6r6VeO{;z}i?6%UYRa=iNKa4bIoaJ(Z%{1?qEIgE^%ytCdt1k!VR_H@+o z+HI^bW)%k1K`4yXRlJyg@v1l_zRt9bo)og#y&Q2R?P&aa!+R=wf)m-WCz_43IHO3T zJ+*i-rM89|7R+#tH*~X+P#b0^J2@KBLj#_ptrY6rs6|W#l`x|H!VeML363?Dk(K&X zQ!?R$8b(`xhx`A-41Aqp;xf)5-sN?%j@gl8mT`*=(y|%Ox~%Sp2GatYdPW$KnC2dm zYHe)7ihgK2oNYrY_W39yH9ZT@+D5V8pjb&II;P|$QCpeD@0WBe=d6YPfsVuUK|j== zhLs`I0k8WHc8c6v_7J${`BUw=(<3u`wV3GhQ*G4 z@*lRp8Q4kXw@_a1wb!=vFu3+Fwc?vTM51afh8i=VACbE*&@b`dLbdveGRjyu1AbB{ zjcNagTIH|xDhJ<9$U-j#awu~Y{iS^)ySPMb@LUDeXcwck-i~zN*72C@PETW8I}5T*Mm%0S%x-H6g8X z8zKN5AQC^Su`ei%JX~Q`Yu{J$%1u^RkwWc z4Jq;K!XsQK*bQ!=M^p1tuOVK~rA7@=-i-^}Ygk@=oaw#qXU!`!7v2YxGeP&MTyiPq z=_v)Pg?K3BIktaw0Qh#vPmY1ZDoJ??8`4ws1MH1F& zC}9-LlR_CwV}`+PLz*%j6RzBmo+Vq&C__5y6OgVoh_`csBR01{6;s*%mZ#+%2HBmf zW#QRN$uTipcbj-b+uQ?4^4lYYAtmfvbh~3tBft z&uM@5FMRVK&@#!|X?ephzzM+*RMz!At9^%RRI6=*#%`oW7RyzRh|bcki?IXt(SBBY zhie^AhwbeUKhs(_==*Ejv!XLvu1-9UY?n(9an zZ+{!P#Z4L%2=pR&M<&e4{F*TtoIvw(X|%K8rqT#FcCXL{vZYz%Y5Y)n%jT2*OF@z? zFJQT1J>e6#LQyH-COs`W-Lf(jIncf~skTe*!aALHUaGf%kSB>J?vU6R#A8-8s)Rd| zQhVsz`RiIyWN7&Mx0Q|Bm>a~riKpSe{ePJeoU$Bk;<=y`yYNV)Qe=vE# zIK^-e2}>i3ZNom??SiX3!=+zc#WmyJ@mrtJOzs)*MV!E5k1%+?s}#P-O6Ai2`!Z8+G>ncrW9jp6RL5d1N7!S)E3IVr;(L4^%d{JKQ2w><_r6K z&#`s0KOY$5$l`M_am;tVm#_T zBLiU4He1X~OJ!WWdt@9Sq#dj;8JCryCcMchA?Cggvu8tGs8$MZW>Dl#Rv6V|Bhj`H z2S2<}Kn2O&3vlSQuRzCfeEm$z5B9bj7#Zr^GlLSOs)i@eL>GtKUm;`fVvl)A<6!w! zu)8f)CT9{N3`e}ho2M?n|^;^kwh;oDS~xBLrQsJ(2MRr{BU zLUqJ=uc%L-jgzkk9=#q)OR-X?M5FjE3AV~|mN~5!dIUsMJd+C)YG`Ua<9jtH47WU z51w==vm*w7JSBa6f)vUwR={bw7Zy>d$Epv$50fJ9e;k*;XlkFz90+E4u-2eKsrE*5 zh#fc!WH{W7@{X5WpxG@sd&Y}b)Q%`CoZ4+FU1)TApG};z?QtGAiG|k$)znX)eUztx zQJ!gRk(BYyIAALlhcLGms?%M(9)RgxK@L}9!|t{iy2MxN2|OV`$2LacAhSfim?GaL zZ2~4@*>lXHm?jAa5dNEogko6DZ@j<3i|luUVE+d}R} zufwV(GX!_P70up(hF@QQ4@u{P-Mn#eMD@aR23`IHah~H|g-_|R|3S}`rS1}M_WMX$ z_z4dfIdH2j(ae_1@KOyAB+DdGvbWszQeSesvI&x~iRNp2g}*=5T`rsAa5!N#7~>A` z5IF%oxa4y-@6({{3zD~_OHe(PTX|~P%&zRN%oC3`Yi5QW++n&Gh6bouLRH7gNu`gV z6TcQszKo6)Ix;^@xi<&;#rHy|Bn?!K zLSm*fGFRT5G+2LvTZ>tcNJ^YmI`1my0L*_%2su7z#Op-xB}=Dr;{eSKhbnK{muYh2 z=LE-ci`&$jV^h{*g!2ml*3k=G!)fl(`3qAv{7{Ly=;`J4(} zT9D%{XGRv{U&;}0(XVV@;ZQLi_2-TI8dpQEEmTPgY<_;!)q6lK?P;f99dUj;Vvk8I z?V_S;d48=3c{9;ifftc8+^D}49C03s9_Ysz)QO`jtf-V|W~RdZQ#PMInS*zmd)^8#4cn8Jv6ZCTu4v83#^?U$nyNXbQJeGbz{ip1i2n zy{Mk^9x<%&_-QMvjJPPOL7O=01+`m;pxj2|-f?5m>*o%Ya?2q&e=i_%hmcg4nhx1o zgKBzP8T;xTYyKH6J9Z_7re(Dlirrix@N-}@dV8hhW3B=-P&xbCq*%tiNG3wRfE^&18>{02pIW8q`Fa-!ZD%vZ5#8%aqx(Mw0qo|qO%t=IglhUq=lTv&dK3w$?~8Ed5Jv{g-XhjDY06F}Pd z9SeI?blePZUnO_f#Dmi5_??(;{++w$1<0pSj=*Mm?H)NQ2fX)O1@rv50u#l?Dwv1P z0o>M8mFYtw{TsLTzeDg#W2%cVJ8WS#~nz%T!aX{q@(Nh4YF(}=jOrO6%x8pzB<%&S6Ysex)!<3>{vS#~W( zH7q}yy!ksIPEmzNH1fm=L%AiSyxh{LYu>u3dWQ5pzBHGGrC+jH((l zj=U(8duq}14j;CapN1% zRyMEZy$yS9ztcfZArh8_k26GuN#MWolcz?b=tb$3wXu$RsVicZlI)4jO{(Qfm)PSF z_lvwcKIf|clK+73nVZfV=L3yp1xaT{UVTMA)pho0Q@76{ZgW3LnqOAT{UqpS%%$%V zGv(v+yDQP-)w_l`s^$E%BUL;U{iAkA3s%$MKU7|rw3n)~=?|{D@KkN(5a#5~=soEZ zwz?4!&=K$|j(l1b{QXQS=-7OOkqJlVR+Ko_B~Mc^0~mh6&1bzPpBt;)zJ8s@=@lE0 zgAQQh(l7k}Zzo{ax(T>$>dmQ6^N7pf>__i6W37^!u$$Ke3Ilv5u4Bhg=XW-<ff8C?K+0L5daq*h-(_ZCj9;TgW*}R$ze)w@;!u_+;({g$7lm1^Apa7pV^_V z={2dss5QgeD%u4#yqmvhmyK=OTWcB%TkRAB@DuMHYkQmIo>J=mQE?cxQj<$$n-1!cYIN<}$Ws z*j3xR?*M!DYM9J=V(h-^(9~Pl$zAJAc->ArM0tJM9NOmZKI`S8J?pQMd)?-B$p|K- zCi$on2Dut*@zp$8IghOI)D{-&IeC`syRc(VuI6tOaeFHL4DSAM!6=_QuG{l$o=MXo z{&exhvT(eec4&DjR%LkPR+YE_j!vtoOcVlZvi2fj0>^yCdlZy2^?TQiYU^&axrPCugtzG#u3>1J+zk8 zmES4c{$w-VEx>g}E#TH@kf-+|5BhbX%gX@9*YI!y?@6z)VE?DZ{~(f0G;h{fOnl2X zNwC1kW3@$#hx;Yb3+BHz*eW^K*%<%5N{+1;hQ(BJ_MYu^>t`=&^3Q|*<;K4(HJ1Je zC_kVUv|Dqv3;c}qET_DSzNyg_zN48ZR(1#|dQ|BqSqUzNL|zkgy0;7aO&f zvJGT%<%W|Ai?ueGXjaccUb2Ze#y#p^+Od~3f3PURZfZOz%$`2AanO6T1d_fq;K`*O zM%aYE!eD;qPu?_xAGsAh!Wfq-?O7f-%@A%}IwnR*>!Cz=fc43)-1jNBu1OL9{O_v` z&1%J@|8E*WZu)68|GVRl7gvuhT+iR6`@j*u%x!J|y7()4!{eJA`lG*o(|t1}ehq)T zj3)V)$m<*Qupi_Hh#;Mb+nSNf`+kut5AN7&Q0ekynH$mi)yrkLlb@IU^FuFL{n*Vp zuzCokj{RybwPqVaJ+WD?M#LB=|1HOD@`ILx%Gm_;n$^8ert{XkJb5$QS3R>$1^i2^ z`wjK_k2OP0N|%N2@K(&)Gzyw*oFcfe1;Yvi?84dAMG~`r{@9zQAG}A-+nY6?HaDE` z*4v{u>a>63z~vola-RuGrT(IRl+SsO(tfq74yoww!V?P$=q*7#=PcQib?#aL$QRJ8 z1G4BoPxwbJ@2AgUKUf-tImJ+C(&wfijK#RYa~L$*Z#5f7%w9O^v$d1!*7J`J?;GbI zyc=tVg`XL0!Sb$VKzlnRD}MX1+dI;X``2c`xXApQcHg(xH2(7s*^FB4ZyMf=#i7S= z6aNZf-mCx1y!kI7>VKUObC0m%*rje$;bcm7*05_Buqc*z zp&lcThPq(J7l{0~-GYQBS4JR0^mz8K>Zt>a*j)a3U+>=7XF^keO}X`hSI4&`lt}3m zGkKIiPOynS&`Io+xkh-+1TDRP1Dl|X%D{TUC0xH4%Vn6MI^A%Y;lN~aG1z>6f)^uE zk5v~{WJp1xh#_5eS1aZ&1Na|kW!jzXm`dK@3j(4cD8>TUl+Z4uMxi`17QjK3+;|qR@xkE7zztz{V8>UoMv(b$)g?5b4`@wtdDCs2qvqr-*HnJtK2?nH_BN#PDN9P3phuJ2fp&!)PdbYmB}6@OX(@ zBOMPLR?nQg6A_aMXda0BlOl7;slTF}v8J7Mi2f*$x%AWP9m9(509g;+m(q)LdCD&2 z#VJWhV><3udqNp5VP^e_Ewg={f>`mf^tTktLL#Vgv)^FsuI!OUl%2N0uMNJ01w?1k zwR<&U8y(^(jLM3XK!zY8&G`0=hr7o~{ z(fxL2YM(V}Bqk>Izvk%k_uVNh)To$>vE*&i0LWvFzjY^d@7{`hvUM_?Gn9~x^P-xK z`R`Q^@qGE#i41t`^O`pop74?@hJ9Bemvl14t2St1TJIm~fo{FmvKEWe?$7_)uSQ@WEJ&81ZFma#1^_T%NZCp^^p?aZ zn9cwU0Lmq8H>)gbb=W5uU#%bAA%;MW%Sd4}{a_ZQj;7ZcPh@Tr}Z zYuEMhO5*9C%Z3Lan@lEvR`sXko}&jyS0YwwrP$B96zM2S<@f6w3~8qLuA_~Iyt2gg zy+j0?2^j)t^Iv1ZxfDBB1^!Z0yIxzJ>&h2`bIfx_`CC$_53RodTu)J>+g*a*Z+gv)Un%I}Zyb`6Oq<+*Sao;D9;-8CxT2Moe@oTeM%x&3GYbE4t~Y6o zn`-D6uId)heF*GKQFuRTC{*`KczL<$-FzQ;_nFF9Kfz)~k!bVp-JZ_EyVF@=SuYLK*$utJv;Ri?$e9GK*y(V`41n zwe}C>49ND|;!Fc3hLo?U{AUizw;9=DD&|khx<~{W+?N3j7Qb(_vCNrBA<^qS4|FGgB9B9F0G6j% zytn1Xzz5Ex9pR)>`3o(u2-H^?bYbF^rTlp7hW9KOt^8J_Ao)Ztj;)5EmBCz zKaXW1I5<$=$qKEKQX2ZV$V?S_n?DRCiVSNqQT`>Rs7-^4Hjt{94RI@DCeNmjsr6}PyZ31T_QE~fBL zg=OOW0A(iEweWHf#44OaEeOQU}BrM15A`;wU7w!@Blm))k_+l^`AcT_ssdH5rh zhZ<>l`lWVmA60Kqiy!6ga?3o7VlbrQb#DU_65y! z%ue{<`vB_I4TC0GlC6s@u&=@Lfo;o79WpGcr5y8d4D+I}doJu6;$cceb+r+lIKfO9gPb~t-y|z&-hC8w~ z_QPr#cIYGNC80KS2Xx}L&7i1Fss!YQ6&U2-WchMPXLIIX*YP{KK8tUF_f@*oRk&xrr6v?axMDpEB+yn8eN|b z875QAL^)k+hhb{D499Ud4sRC^NWCkoxl>I!$UasHkcS^^d(#T~+!~xKWWWpx1x63k z?@BwYVQEn4AbdFGSy(cVqOkYmUvL(51YxZ*YCRU!9+8l!RO|%Z`Y~!(>FeY-iEx^O| zAf63~5ufK*qn8-<@{+3^*mb!p?Z)924f+1iwzt&teIwwf&-;(CU+T|h--{#EI z?+i{j>LozbB6{kDYSO6fvZoeKoS%`HR+goY*dt|gWPvdhyFHK_FOJCoxVJ@ZNOSiA zWBRK>!B3!d8fxAsGhRi|IIbxpJT%!(DaKKe7Y(ks=1MMvIh{%f2&?hWP9vgUqF0YR zQuM0UMC_JLRTx-*%JXL}f4;CkMURfu>+>B4G^rO}99$|i?;%;G?HgCCRb-o;1@ElE zPK);wh}&=Xo5EV?`a3otK13}0VtEEVyqb!9OIzNq13we7eotHD>di#e{XmY06Q`8t zt!cQJA^C_zl~NG!RKhWp%NE5mtkqcjrbW2%vaxz^aVf=s`~~)=8cL1eQo`j-7q7xA^-^1if+}p})du;%LFUb6I?$Xd5QREVCfOl4G zg-Jj>tNZG@jzeTMEe`bwdVbo)TjTP8v?_G*Xlj&lay6olqes5CzCZ7i_tVY1+2g!hi82FYJNsfmoobzyR3ZL(7gY@r;n{Y^6YIp26O z$1|VdXH-ZcpJTMt_;xg=!=|T)F01wx7#PqaJg9`I_b9^h{iqqmSWs1qCTKHwQ#z{q z;6#&ajAI+zzS~M(^b4UHbcNgG?O{f(-Q7vdn#$%q_Hp?b*Wr!2>CDHzW*AIyd9o~? z2~yK|Y?yXS>+h;5+PpcRY*=Ez8K=C%gsjUAzul&%H#cwz<%fhn79NYWvLOE%3(2Vw ze!UI2Acq;@syd`nktu$074DLceA8-*PUWKHYJ>Y&yk1hYvn~jN;9om*_=aOinl(|BQUO z@!0Y5eN?9!NP9L4W9UA;*vu@$cn5<|rde%Z!`d!w>TbWt-_!kXCk%ZqZ{D01Ph9%na+=H<3?E7?2=mzetx zZ}llE?g~eD5Hsuy#WEmktLaDCS>iADf|zUTT?TREG3&AediNr%! zxoJeYLyfZc;}1Dp2h29bnKLec(}BMtxg)E(lku0KJT$-o`)v@@ecNJhVH{*lXNA4* znlJWFF?b_|X56VYpDjYp`X)9$6rAwL9;n??(PzUSq}|97c!6$Q(8{O3cxHEd&Z+S| z?V<)Kyr@}?=Ke0TU%g<#&#F)KydYO63rvHCI8@f;l18SlxahHlaoS@KmqAIS$V%r? z7Z=tM_g!5(m#tPW&;Mj1b@3PQ&KgH|SA=bfLC`+g<~11v5BlQkgWw@XxN54}RBil} zH!8rq)TB1wcIi2;%5fShtK5s57^0U3=iZjSR zQGO>#2H)}oAwyh&NdA+8up`66iyC$l0tz%H^fvX=uM(n@AzG#+9ehGUr$xM13^#UE z(|2`sCa`V6-8-D)b&#f}*}k`Ru}r%YE8n+b&k#^VFMrcqaF=k-214Pwkjqz5n)pa+ zSJ(3GS!vj_3HkoOM=W#IGjVV`ts@xHF3slVbZb*n7r$o)n-#hDG4QqzPn0#*ziA#e zxNP`K94ZQ8*DC51pHL5GSa0A}FtVlDIt|V-%tHV-SJ0ks#@IWPo^9>~+V&Jd6Zs66 zEd|HLP&?+-AR#ADKELJo#1k-o4&{|XJF~)2QJ`<}v2Lh+3j!L8RxhqQa0~qm(r9CP z5k$CzI!SCZB<|>1N_E2*W??DZfC1|v(aS=*jR5fP{NOl7%YB^I93(~b2=zc#qI5Js z_Nj$whM4oWq53cG^?kLaL^Kbaw82S=;ka2gNbOcr3V(LYT1q|^5pS&SJQd(6q7^on zBHlfI^))@F@_y{-Mh{Cj_)h0h&(C96ZvgIZr@Hnorx6UuPpGZL02h+_PXOHyLr$ws z*Kl#dzJx7P+)!8FQtP01WBWI`b6KzNl@8?z%iCUIEd*!kL0d)?PUSax`Ryb8RJ68D z^-fF&c5}o+DzYbexED=aFud*@GitoCD$v3kSbAUcRZ<@|F;jtiy7oIc?QKb^}9Iw;|w|P9LM8X{s0S`v^A`C)YU^dheCXHA2^(EK!7KJd{MDB zwqGi^7NG<^>PyV>7|0)AM@7dWoBS7fE^YHprxkGH=#WH}{{v{<#drtGdWY^=PhSA{ z55>z?CU2j8(Ov=?261%mt`e_mto2225n70XHfo{lzOc{uIr=M^Cf;-jvwurN#dowIK z#*v#^KTN$)#-<;ea2vZnspiRZ?j?qzN-C~?1>l7$voGckSURqB;NV@AR7{F>^(lvDAT6c}225AV4I`@Vxo$8g3$6j9myVTD9? z3%z74o&^@w-N|(H?JQkRsEDZ$mifD2e9G6-P0Cb2f|>jb_oT?+4sJ|FCq=FZ>3q_) zDe*yOM?64=4Dhh7zl%SEGUgd}gdXo}R z3^@}|>Aq^5JeA#BqEhF*k<`8XyI8Lw7_&p2CftqPy^5n_SzCX0)YDr)?CgVnXE;>c z{l%V7>!PQbTPXh+qJc49^O+5^n(Ccr7J|o#4VGpd@grjr3T7x%{8LejJSNNX{1cB1 zisF@Qp&50krA5upP}D!>J$;(urf$&@)GazW=|UqGHVDvzO{N%hw0U1$FYpiMXULRZ z9wzlBiO#WKK#0HjNS#K>)4f{d2|5qMJQDMn&m*Ca=SOxI4GD3 zcOj131BX2dbje^20T-Sbb|3_?AaKo{^%KW1h3pQ(Gl6bo7qrdJ>_~iOxW*CASrVM~ zMFfg#a-di<3>39w_xY($g43W*s)5YU-{m_UmP+bBDB6NRE3M}Q|-#6y2>r^e5IZOM%~V|F$r&LX?-NJ_fvg& zEIz2HfG4059<4hNRbd}(7}SIHFPw^O2uRb1ctUFEr7;UA)C(05gb0FK=)c3L7Y9sy zd_|?Sn;aQ)?7XX=YopXR%S1C~+UIEw3_?gqD{;EdE^9aBP!h7KZimdbdkh|ZF5;LB zmj>?`64%Fm^fdqNVm^EFJQa^wi_Tf-W47Rf-NMMTXYVh$vU08c!|rTEtGL?DTm-LN z`tvw>m<4fSv)s2A)slp@UkBu>$VYdgDsWeUAVW<~6i=!}C-hKHv%kKsAoSzU>#}qh zv_z@2kW$-N^hKInAFA3T$%OBK<8Kc9LG2KqUW}b+po`8`mf>qudHkrV7eI*E6Z~G9 zg|cH4y43jtjDOWAS?aQ>kyrpt$A>PyA7!+DfgnyxT|&SU{QHR)+JNiY?X%AC!aDoM z>$(|be=})&$-(Zr%-S@!$mHo!fS`R2(Qv98Tcdgu_hq>9JTi)Ow0*#4ZbK$*Nlbx6 z1V{5|by&{FAk-gNTbf@7N=-%ToX7hnIF|dNC%(cS+j0i%w|>Ak@*EFnm#6u`R94g` z#lEYxz{^;|>c9KZY8LeL#$}L$^1&6$AP;ehv2B>&m0KZyVwx&g1#xT1q)GNFdxd;( z!9gK+w?tVcG|b8$+9Ve78y+{|00Z!ZfC7L?nWosEBlN}kb|;G$c-~+VZDb;zB+(wJ ze18@>);-q|lveRYUC&rEJgJa80iY=eQ(*#W`0b|kj7(VlqiPa(-9% z%)Zq><;|*+pZxR_)qg6xd?Zdj?Z{ceBrU)7BtLx??B1B=zyVGk* zS1jbe8ZQ6q(mLN1eEuyL{cnMvWsiqu|K8n`TlCFS{PU_@{$CsXuMPgM+u-+Ivwxe| z|5bS>fLNdsek`3MLbxs+95Oj!VOFM6aHtHxGrx`unMCqYQHsNloeBfBSs#F7hOvkW|I?m11muN%yaMV_hhLhcjO1L3tW`~ zLLg-N=ZWPwTBmK$&M);fj=x2Kj;YmU@p~AvlJJ%<4DmZ9+9?oj$zl>Mv0pb&3+ zP=XeNL1(+dv1Pj<*oH9RDspy(L3-MJYnt%6^Z$fK-T6&5JM^0R$-zF&bj&?*S%7MR819g%am9X^DulPAH#v3p4^(o)F66;|rXLfr~ zTGy>Z_M3DN0qq>S@&`G=r&kxv{nQGo zNy34(_)hIz%t#$D!L}&hxU6`Ro> z{(^S7$vCJ?wU_IWC>7TtIb%0$Iqv4RUED*MFHn^J=-Bm82dMC{jRv^YH9pq0I9<1$ zAs6>@Zs^J7lboZe&3ZU`dn9V{P_pF z6M*Ylr;VW5i<_B`SU8Cz@4(msRdOEDWw3*6dxyOI58?;uhpf)IPuR}-)gqE!b>zx0iR@7@o8?`2Qo3R}T31Z2 z!u(pJtuY33?}j37gphlq-zSb)$JQ}LCuIwxeErfWOva9}52reH8DJ2SVim}nE6lM+ zUaF>5Bg~OehuAub9lQj@pvf*!fvRRtIu&FDHv0L8g{87>Zj6~;V%_V?e;2Z(A3p1b zliWrQyeYDIDyCEpt;%Lf1D(2>= z?Hh(ISW2$Tk&U)ShOrxsxx1)AKu~I1P1E(y0!Fz3SD+Qq8_|Ok;#!a1b`JEZL;)wx)a;7=rITJ{c>6I0`_q!4eAzqAsvvazMSgY z{jifZVIWkLA-yj)&vSH_Tj3995@46HWx3) zM{s>W1-l6~X~eCSwl+c-dAMUfd|mNunS11f3#Q^^+6&|2E$^9K579O4jMhb6`;+oL z1w?&~pkwcS=ic6#2+DW6yAWXLg+|;J&z}f*)s=9;-?+-dp#*biG(y91=QuX%o&GaH zHKHh?Nw)I@AupoxZL6PgJ|}$j>05ZR=uw4@J8(p{`<&$!(v;weDZOBGk}6r;BuJhk zj4DkVz3)HeetjGHrh536{RK4rU{`$M<}gyL;V!ZJBgEiW0l(6KwID=Vr;J$(D8cNn zcCo6ihY=6>@mI#BUm4oCP~@@Il^IhCP&O1t^f4e#h4=gRNDreDf}>AGq^(P|8WMu^ z>IA2>xXvN8GQJanWUJW8EtOZ+I&Kh;o_ue2gupIk!@Q?j{kEAU<8PBR8=QP?Se0oF zYl+X4NmI$O+yW@|WLmV_@ja1!E%OnC(k9G9efFS!Kqzcq!_;m})s!du=lf>brzuCO z20??qD*13`YRjqPx=>fNS3`g=on4vQVD$aVw1qRdI*@wHy*fV@@xP(E11ZL~kH#qT zy{2JyV8o7Ot)MyYhIIgD&0mK1^4T+-bxld$@W%WFPc@PY%DIo2gmJ|AMvE~(I7wUe zYeJXhdCVQp;#cP-^+mg{SOu6MMyzib{q>&TQ!2K6ff~xQJQB|(yLuh5H)f+wb=-(T zS0st^G`4nVUsA>-k*w&Pk97q|gI~#=_%$G(95cD{W@mDLkT&qxZtWIc&hHJs_CU6m zQszv(TCFoR5aW3WjCgA#{NlLk*bIkI+nhEiD2@p~-5vfm-7Ts6cK`<0`7@Ecz6XNz z9+4Cy!j5&x?w5A;nUxz9YnyqMzzDRyT?=ZTVZg2V04bSG?c6MzO35frW~4*!5q7e4 z)Ms6Kg(uQX71pxS2)VF7hU#h{$pp;@CXouHSD;~c3{l~I;(RH(J6{i@NV>v#W-@E- zvf$Q=EEB;_)2_WnPtKp2SkFby1hyMH(C6c3lj%&~YQ(y5bdkf(b+)C#>z@+`NWmRx z#fDQ$K-lN}mmQj5m_W?@E=)ChqdxHMV$!9s%@~XI_9uum18kZP4COSmgQfN|tox$8 zJp@LmSDV+^b=yjj@)lZ(l20I6X=gpNw`5V97W?oO}l>!T;#xF@#={BM~pv`#pT9Adp9O=yRs&*V68`|x( z*vz{E8J9;z9LF!$KBtVapdmOb@81v&d?J9!ji`zYsx>JOy?x@rJ28~=wg_JRnv#Ol zaxh*_O>pLnZrY0MYQ90ZJZOWp*eJaJML*<$HgWCerErL^Nn3oEcl;VK$ZEW)fSo$x=UKWo-&gc1+76K3UuWi z4@vj!H!c?Tke)-pYAV9(V7I#Cq$x?o`N1oV;SWU;wt8&`tMU>00%e$4GswSwD=IEm z%)7nytd(i1LmNb4=pK!NaRt%R{41lm|3G&?7)VEG0ED4{7+qx?>a*zM@6?u6RDt4o z)M$6w8Riod-XD5`y4U@nBfsJl6I(s+%=}bnvY=2W8P$Y=pVasvXwNjO=tX##i^1;K zP4tolxyEwP7==U#fhf#T>ujdU7!!Ht)GgT15Lp}JH4~ya<#&r*kDcz;21dGS#kwlz zK3#E4{3Re^%=-P%{5|LHpIsWoX( z2!`;g{3h(fu64-_Z;WFSyFHKA3|cxM|QEW zo5_6*fHs79eG?^93`aRL#40`KtAYW{ zdy3A5A_ViOAMzrcF?!30{Pa%Qo2lY7lBvuT%zTBRQT4=c47G#fjUZ*-<~ZERS=`K{ zKE|6OyF@|>;bT|<+QxkdV9rYUvPgG1|nceo(NfPH(&->?cI zKzM<_!jcJHf+xajqXy7o1S74*VCoRI%%<6LY)aQOTDwq>ZhJr{17d*I6B%^ht3Rhs zE`iaZf|H^4qih)3D%Fu?mTpVokEUj4T#GEH6Mi1Y9b&&vyE;>gU^iZ2bwm4XR_5rl zA|7%FzleX%5AUe|x@tlPb3%HM-DcMZgHeNtDa4S5H}i*NwXC@h`Kg>QrJtg^oJD0oOn+FTv$qe6y}hQXz70LLi)nVQfOIPmSX}6coP0KbSI{ z8eL3FMwN|){#6wo>eZi-*2Ud}t1uEtE}_11YS2Ib*u>^{jh}7iCYvtKP$C~N-&xpD z6Qe1qszxa7gHD^UZmFj zANJlfEXnol|81>Sxf;yOQcKgyw87NW6h~+>D@&U-CrT|-L{f4@q)9EuOmoPA(#*_( zoNz)Y2XLk`ML{J+MMG3XK|o+rYyI}K|Hppz@$c1s^MCF)z7US%;=0fKy07azzvt(B zQR9lAEN#O7N-W)Kx?!x`r{vK=@wb_GOXVj6&Ual^vb*V$ob;Yx zJbtVc^RaTwPorXMaJ<;dGL_un`DN`*Na8VP5EZzd$%=8aKNpDV-zWIv(iPPc^}rf{ zn`PqbY0T$bGxwH~DOdz(vp6Fu%5WfbV!0={2ikoJ3-BmLdd?^=*8%tkULff+C||AC zH&a@9+zGa$YE4zI@E8%`#zJm-@g}&xr}e4X+I@xYFIE%1i}ZR4Ed!3klOhMvk3qOe zHDG5Fww;qMJ`)JOcynXG40q|WK8ZywS73LO>}X3vcRj3jZZLf#>j6EN{StojXaadW z$_l$ADVT9dGQsL0-CDGj%5>?J$ge;cwshrAkNlO%vtNRVSo71ugU`g

C>k%4oT6 z(sPtIqI2l0y19XsY+Xy>oNk;8bTy@uV2Vn`E|9(32nTOkx0bi z(uG&T?T_W2SI3NECMgt84X?7Xv+$++1DL|Xpw+oO1;A#D)pXx|O-Iazg!=d_SNB=MFAjM_*HR?=r7u7}`f@!n~l3#s>|$xhIeiY;hIF=IQBbrSaj@~jeO_@723w6XCB^KlOB zSXZ6n0&d{NJ4E!4k=c6QN9HK`grX7x1t>O>?qHSsbB~MK?*9G%1Oz-Haq!DN*38aN z7sWHL<Eg(7Ma^9kjjGzhO&~GT*bocu=t|T zCp$FpLWQevL;5zp^G|5T@NtxgCqZ0v0_bl0{th9J@Azt}h-lSB@A6eZe}Xf8+qAaI*vNKAKD= zZ+|K>SM=ywu}Cd!?RmET!z4n+J#HGAeaX9#8DyxAPI{{P&BI9D1f3M!AMQ7|odH?z z3)+%YqUR#B=U(;Q>!%d!-|HXz#|O3nYz{X0033O`Q1iz$c+x52yPQI~Kf$o|C}@?! z+=Sda_aWuK%cT}A_&akueyMz;j$a%e!;lO1)ERJeaC@ha<=BSm5{jXoX#e(f~Z zsX3WGj{_5pZ6Y<@iYdEApFw5kJ(C?+p#`6}=61detzYxZ$bEFesf!!t^r&MXDkwe_ z|1bz`{BSd14!#GE`lOJk#;qDSkd27GD>HYiesBGbGtaykRhxEgy@v}cAk1~-a+B>1 zE_LC*qiu*k7+6K9p$>RPe(=@oqJpU(WI}p{IY7ssZ~9tK?~WYZ4U6?8+k}p$vm&J{ z2;t=qvj%;Gdm03kA8Dwip8YDqFq@E5#k7@O;YTno1h0KbH9b+xa-Cd54eZg*t!XPi z**R{N(LvJ&XomL|8#?(GEy7ij{kNgSs%1aP+wRL|D`KSwrAgXXO`<)@l=-*v)q>Sim6Jhg*+kUFr2Ej{N zH9oT^u{~zFB6JpG|4lOtEOu=~CEMatct zdn^C`oMG%FxIP6_xTgUWNo>N&U?!n3AjRNER-$9v{G~re7SKd-1~r#*U|;%%4IPZS z(7hPP2WEi%I-wS_m4TA;0k}qxG+)5ritbR4g=9B6+#|! z+vvsnHE*C{IST=Hf4J{bR;*&I{l>G_prN!F5C!nxqonL_o)lhal^Va{EUBprU4bCkm`kmUg#PZK>Q$RZW@;B*QkgTv=y$Z=Ywc}=vU(6A6 zE5~Yu#?@yTOTB{XJVH`wUzqjeq=!a-q9tqK$C&;=OFr2~InE;6Oi$M13eoVpd3EI_ zTFHI)HDUH^Gggq*zTmV8Cn6V6BWKdz@>%a-$ZQRNYezRl<{FZODueygMe=ib`uDm4 z`U#t9iY{-nv-Uc#;F}I^cJH4B5M%ORc%w|YQw+UDTmWHaO6E)sIU{-b-XUU;|mczCLS4I~Q6U9HxB0;_DK)ksQlo(v}4Q&6slyY(iIiDn5?f7c z!m>gqGl2MPju>9*`G zG;NdW6ZohLF;X~`D@E>?z>-F(glMT6^9&;hsWLM{m3-kerY1~F^&7{e+Ze*xWOXYG zyyLF;x=MkXI~Udb53eKjH&X&K_{XSsHgoJ5Q*s5aX=7D%fyJfNygMxEYc+bSx?i%F zXBQhvtc^E~M%RJW@)=_Y8#Tm%CWMt-Cc(Oke)K~nuQ2GJcmwMEW{T-2UB|thkoQ8e z7InQvDelZS^DQ)@>#F=Rr6~P0r)48I7*)Z>sMpZ#9)yt zrcB9mSzgzx3Vexfqv+5ok$D!p;!w5e1zhOHG_!qpO8G?sShwqIz-Kl^xQSx@{E;2r zZVuD^2|flX5|pJ$%f%Ray63%Sff}uNdCRZCf7kj=LXK$bsP?7Z7xg!Mk`i*o%q z#q*D({m$i4hdI@Ud<|O|*uM`D64xBgn{UP0uQ~h{({Ab#@8zE5kxKe_f$hW980qx||1s(=5Y^r@RBTs&^}W0(}KCv)%)*MX{!w3a8C+v z5ggUAWly|ynXMt#n948Q$`%Ny_Xonv({>Y2lm2q93T^Zi{8ioiyo{m9m74uqj4*K?%2$tiL5*!p3=GEB z8Tj&Od(ev(D;3v@Wux*YhUm#grJMG(g{K_&VI-tN^hG@$8l*uFUHK$%aV?%H66s_6 zCRVB`|E`k1cV~c?6y0}in|<@A()G_uPooY9oc3QhBdZIx=Iv!#96JorD4jS?E@2J0j^|OW^OR$5YQ@6n<*RI_-h*HIll)i>Kcxnf7K)tn-Kb9 zL(7gTm`cKnl}C?7X^bJRmQ6NX_*NLZ?w{X(|973efSU+-cd27>8O(Dz>H`(icq~C9 zaQ?k(XUI*B&36CXdv@-IqK z%`00q9cpcj)GP5_1Cm7PA-ZXZGvombZlgnff(Q}rAo&ztGzXC=dgei2022#pAnPVW@GqT-YA_sG?Am79Kc0KFA`RyWx0(1<_%v|4A@u8Z`94<7(( zl0tV1b5yM~@V2T|5ooX!U#rgxhuS;W!v=WbMgNk-*Vm=%QBZfybPD5sPda68@ME=% zK63LP`iKm_r8{`tDsWMyP=vhbLfu%u_6ywsp?Yt^;M6B91a-hn@(Gy}!)^!9fts6RYpe+*SD|{4P#&T-%YD67KDIha7N>2!;asyRi6+l#bRlc@8oSVl-|J zR=M)Y(D5{9y+n0uU;%U=m@JM5PjWosnRKn475KjsXE8SzLp~Fafss7x8A&xJ zVLPK|?)*!JxCW$IQ|gTnLMHOa{i1vbXCMO-&)KoUF~1Wu^IVfTV7*|jTsCd6_VevN z$yFR(HSk^AgH)HTKy?yTOsthW3Ef6d@Hoe01UJSqJ3~n#h4NgJGEnTV1i-;<-@Bt= z<=^eMel7UZ>Y}R3`iZy4aaWuGXtcXj!X|Tq`a?%x)v_ahm#t#%H!O0IV_dqdjv#vP zw0ac8ImgKaf?o2+Cs(2t~+U+k#@YHp1v?jqa+eJ7Xp$toKdob9_M zzM|8%W{!CmPLe>+tRxc|l?pB_Poy=LpUTtNJWK8;N7??9*^{|0I{Mw0Z&BO98M&vW zrVaD!P%Dghv?JAY90GP_%EwK=()A|WfJZ!$tjb)cUxnj4^=&}z@-jG#vjdNJU z>o>-fcUh#{MNc0l0O4L z03?y(dM^Rv{iY4D=_?F@wy^xk@curS#cj&NJxQlTsBMhwi{)zlVF>r7P*sE$a5g(5 zaPNT0iYR9B^@{8IJD^Eys$0|AoVfbZ8=bUgoj$XJK{s`B0j}?8XXm!bngR`I@ys~W zw~)S|_MV|R{XCQns)NE66`@VS_F}k`^JKBwHFcL|OphGlre>2HT$N^zX18WJRS>Gkns z=Ix;@xXJQqEYpzafxW~$8e7W0{sw+mY<7M&7Sg!XI+==W^aI~^L0#U6%Z|e<&J``S zEJC>4*L6zI!dRtisu?~rec1s-bHR;W)T?6m8!Kkj+U)11Q~awgp8C|F*Ec$@4_{Lc zG1O2((QaFT)zM`^3y2>fnqLSN4%4g^_gQZnjUraI=xKpfkVws`!r;O<@t#|3>kw=K z-Th*G?gqgcq)1(yjfq1%X|459Q`&$%{V=J;qyv!0J~DabNLCUp7$V^kRZde8X_}A++QtBmUzqsFe zg?H@5JKPS&_eQ}NRM;Ji@^=nfIp+x#e3D9G>GjtGbDeiL*@QYnG-vJ6H6vm>R^_Hm zo=eQw`gBhJi%eC!Figc}7!b~@322B`%93-OTUfu|>R=(sltQG!}owZCOi z<;%Ckp%qFzsr9_0F9JvizdZS@Oa{SP%ZUfGx>(2TN>j%- z*3qH4JVzZ->751}zspn*@nkE!(I89kp&B#|Q;RcnKcL_Y_uBad#V{FJGEFtO`x9INcEIF}rqQMRJ5<4B39*^wgtm#ti3-w#+W`&k5I#{{$!MyT> zzJtVw9JVIu$WpnXl(c+KLgs#IftWiqu3!e3~j`zid4(CY@gEHTpc<)Z@4-q4EfBEn#LcqD8m zn!`ToORgdW`>lq|(7uBG4Dj}pgY>p)E#cMWZ{1?$PZWm9i#p@M!;^X4jS~#rF~*{$ zXqU*YN|B`3Qxkas)4IHeII>3cf~;K*HOJrIp1&h0fe>cR1Bkq>+__+0fMRjbN%~n4 z$#4RQoh0<|%aP_fYMM$gfK->at?sqQms7lbw2n8NJM_1PgN9eeCKIpYl;23=ck;qH z7HEvvk|tCAeO>nGiGc%zaGmag!F{VCfz@MbQ|Wyo$K7=0$=)#wH(Ye1m2mSp%2s zUVd!8ZB~?1S8d1}wd`gR;_3N$xH+X465wWthblqK867wGN+WCXjU?028`9X?qbCG5ez1kxxB5OY!6_dUcKHr%6KO%kk77ST zTHEh1cI;F!JFx5d)?%NPk@FvZAWm$I<2P4Ngr%Uf@hq&`V~5*b=&!tJy&pp-Ictvc zC_iYNWPkF27HtECC`|!rDj0T3v7ScdnTG>30^r^c{9r4X9gqg7L1Gjr&Z<|k{xM`B zyMjWZG%u%qeU(jVpQ zPP=A0@)c~Nx$#{FiPvqh?=v^guQTyef?L(TfyVDHN^y1aWY7lWIWvh%o6N=J2Bl|( z0^h$#*d3!TRpA_%NkLYLF0)Mn^P@CapN}hUDD3ud51nGY1SUjoFrOGNWAtWz1zPIZ z`E;71O)`lUGx>K;m=$^yHs!!am?;_#NIgy&{YiV2Sy^kH>!lj<>n~_O^5ikIw0;fL zkPDPgb);afZ}27AT3*&avrcLzc`vwOc5tB?oaDV|rAOQG%+u+~&#|(F{Vm6_quruY zk)aYU*HW;1pjub@0z{HHGM_P}M!g0+lSbFd`IQ=f>-?mufSHe)qUIvdHw7Ydvo6b|+op1^xQf zU|3IOG^(kw{BzPC+;N)bY`@IFB%=S5?w~SRfcNwfJK<}T%FBi@&8Ao~$vOV}>1(a} zk%c>E*NvglnGUt63=lVcPD*JR8M!=#Xm6oPW_yIHXEi6P?~X<`louNIx+krk6$6J} z+5%Ou$A1~m%OtuE^f$z;^oZvAyPjKn4E#?G!<#w0gsU?sT={bVts`w)D!^URy+D<` z0WU5u*4{&0f&fC7qB%44U|I+9;WvK`0?P9sX>1u{ST?C+mg@~Yx1L#tjFVqf=7blr z%_Ey+@jGHxK-OxV#8XBFY8YBORi+saLL+0rcPtmxdeDC8h&=X{;CC5?+$w0kGW>)V z_nhFS4b9yz4wnk_CE9{DC8f}VnJa+rgO%$c6feS4@CNh#!%6AdUcxag}aB%+%g)Uk(@R4N%$=EOY>?KV91fF6@rNtvf_KKY4z+ zX7Qqydzr(&v!WHQ$_g`K&Zp!K0$vtv3aQHzm0+~(R~)y{y7600PtvpMT=V_uT~x^x z#Cts!U@-TyFO4R{lbmQ!zLMC0bW%s1(M(0Qt$eP#v=B#!703=&F6=6_F!Jn<-|VdgM`h7Td8L4y_MUx*qFN ze`@4W7tpu0lRJIR2aS?Emzs+FR`j^B{HGJM8vf>VQ=A0IjF$l=wYjs&F7G} zqpRB@u?5TBTtqzFJuAAlheUBjZTz*~Uj8YSXb>(na456HJq`15L9eIuw~ZDw)z|GJ zON??;!H-p9rEggWnslqIF)TD8(=jr!$tbDp@~n5;r4q(V9l8TEAgKavJH{k!O1yn_ zhFF*1Pprl40#cgIdpW{zk<(0fNUXegIH!+mK=hkOe%sst-d)xs=Bdt=LKv4^d-?qf zvYmT(KAQD!O@8*;<8=SDrtXXNoNRx6M)yVN3A>^AdKodS6L5l~b8zU@M3zhMDO@J? zF4xuHcPeY5R6Lz!6>!;Kqvg=>M*o{wH&IDnu(&qSdm4Rz#zCZesU>J=RW(w7{49}8 zTP}}=HJlt6`OvE7J)bY!vda&`V=opm@;qN6RfP&n8_VOU(Unm`f+DZVfv?gAxmIK>Q|8Y?w+LUd@LYyPKLqB{RN>Y0D;eCYt*teh5R! zM0jJ^&>F^&2DG;WoZA@jEVwFB;V+R|Fm)O7Gcd|)RyHRUJEfM(i$$iSh!Mc>*dO2%ejs-y2F|Iqv{yhvk5gxMQU)Z> zygqpTsS4d{s0n7!J?qv;7ap9g+iUeVlS+O(_+P_Klgc2KS{t~wGQ(5HdDyhvRHn?D z=|`oBo-=*UN=^un-WR16f@hMy>tW_GGV-=wQ$$n|6jU|o1#0u zXyqm>(nZbr7lScncQ!0Blgp6ri_T&$#9^voE-2olb&V8RZ|;yP1|G#u#NaPNtKC4k z9wx0Vw7)I@?~o4jnErdsC}-3BGbqB+x(7Es-O5{KZJ(n%lEJFZNHxth_huhU7CDD` z@oS}L5*UtE^ zlcMwBdZ|(5)g6Klb%eaeFR0)~|DNQviNnduk%{T)3P!o5jM0J&mknDvu<@jhPtlrH z(oXD*>Ux*0(?ic-@8y54ttj9igPMz#jx9BtcLra(OYMbQEY6L$Zb_8Zv=mNZZh^G` zzK2>(@faeEdT6pEwpRYlkn9jzUF3a+Qq=wCJy}_pEGZL^)#xXx0xiqU9T@}7CL_y_ z*!}6@RyOfhXK=NBcZJ4v&pdGTpQ4d8Q6+oAi6y!Qy5_QfM&s#R=Mb${d1}a25$&Px zOS^~oBU4ikbVr&Fm2WdN!|o5%FE;H`y``3jie(UU-^#}MNZ!O!g){b==`ura-=ImB zx{P^>AT&pE!g{;%o#LWj<2-BZ|G<#n6w(w6Vg&8ZCCb?WxuP$!al%r_zEy#U@ZpZW z;KZXS8K1`arR@oPgtGbQWJx+|{xFNDw&eW$8JCjP$*`zBBJF$0`M6}T=cl1i-$GWv&rK3UO=__h>+ef0vUjMsGp2~thu3+wH#0V~t%?&WOzk7%^r#Mr; zA4n;j`>SA)8JV|JG=X@&JVKPc^0l}I-y`KTBHoE-D{osTuq*MAP8!{vdjGKL__|G< zfWwc5wpRE9fkZ4$~4QsP= zarQ!G0<)mBfcEpJIW~KYc!t$5Fb%tQ{M1FiH1?T>wG{@AqBg3 zL_5WMkh7tmXT&cMb6iKYXz8??#pypsaRr9&P4iGEtz-&jk!o+O^a>$qvj|Pfa#ujw zBmKHimeCo6ipFcXFJ)^_k>KJa(dY%-1*!}Wv>Ey{%E0%rX`m)n{OSFH!n2}p5l-8p z!VNF1S|wdPbMlz1q78(6=b4CnWUqc!G2B34Q&U9BpL9Rk+S7oKUb-5tN4i5vR}K20uo zakq!PDI;xv#6_*DWYGt9T=qeB3isx%2uEQ_&v9_r#$tG1mv`+NSx)tMu8O%4b0*s* z;b)#9nDC`clM_^~+=F2Y!u3obr(Ok!FaFA`UMt^hzw$HBkhOu;gwD;8;!4EuN|98( zbPM6J;~iO`&T-#yuFJ)g>*tzAk1(SWigwA;d1W!Svc##}98Cl>i?W?l{OT?I%J+pe z=3G}2g(f!cyN>gI#}5g))_Q?`bK}7BHkN0cFx++ML{khSnH*L6FG}%HlO5oUB)#R{ zBjcerr`pwW7H%H_9r^n#;ak-`hSgzS=GLBGz%{`sJ-!bt-UYKkWXf|K!7;ntn73i| z!B@Hy$Ab2V2Jv7R;-iL&sY7|zNT=t1au7QfTBzDFaPX<=jCuH>BJsT$wiuy^QC<*! zi=L|cfE-koy1o$YcFb0-M{dx(2YOdrp1@c;wuGndaCnqAH$bs^ z>r9)#!Kp3_$#*KBL70N+{joBM+#i!_9ruilg zoIfG3aC5{0F15!j#wAh-eT>$j1)1NLXxmJu*hQKo?wc&OQWkF?K_j);eLywm@VzSC zs&S^oW$nxXd!F_LsMG|ECfzP`d9#?V&2+i)f!5+O!BABLB^iB{jyS`{7=nmUP|fvf zhi(Py{yhd`K}DFY@Zq`m#X1HECW{m0`G{&|Mx~Ey0Wx@KY1x9|a(o%K&N@9l$^EEm zRyrcgDqzNFtxj-CBg5RL$0<)?eEAY8MgFqJ6DuWlU3t+!O||R|@wzZ0#(wBtNxUYs zq?L8gZP2}G5?KLQ`r^D{hV$*3F`ytR8iI;4Rb~`%wQh)aB~8>Bi%wA?5w|^S)*BP^ z64VJ74`M5k7jI^NQQ3DT~Vz;!esVX4Uy*s#MJcnR4CB`+FZ`RrL8 zmRx3sEbx2}emY;rBm~+g&fT)iskKoDm$nKIB^6hZ;~Pdnun6E+l-*RF7qTk$&YUOk zA?xRntxLf`g{0nJ=Xky5CcuTK)<T*LtF|s9k1@}o> zSg9Q>G{*6A>VXQ40|@#}XZ~K}gfJBR8tHIkNDlI#xB#h_>N$p-b8}7VX1?FdUzYL1 z{_A8`@ly|6wh{aT_#^%Hss-u6$8t}q>ibt zAf1z^F@RW$?~LQJ{}z@(_RX_c#T$B>F}v=<;I_Z*Li_z`20Bx}^+cNWA;}_tzeeq! zt8i&moA_egP>S|e&+jg)YX1;eDx!`TpPjRb{jl-(#RvbW6@FJ9_rJ(rno4I7@ z$YkYfI4fv$$-gMpqp6_}koOK$|A3DMxlSAFMA(ln`w6-RETWgtqz4HQ#@islg9y{Q zl`e9B=Ljs6yLUN^F*G<{+zr26mJFLi4X1kQ;nX*Zp(|aH1qmunoe?YM%7HgKaWw30 zT&{NyVk!yF026F2;AO79x&HMk1EFZe3;aLHE1OJ(0hc2SvY+raN~A1fF82*t`S`l1UFT2_Z6{xnRt;q+gIBSfoTZSJjmRV2CKucol|ZG4Mhi^iXs zuDybEY7Lf*VF#Rv{9GmT73CRd>|rA=m?>W>$#vkZl$OgpZ+2kj$a?X?elwP@;MJ|h z@VZZH);sERw<|?(UhTx}Jkb{8IDFzw_Q-RHY5Ecso76e}(55?Oypjy2=PgpjC$T-} zxdI$4p=L_`98|$m!3eE))!m1&rr$`@nDj8`k-`W*GSOLdT{09p%gp3z+)!IWyfJ2W z^i58OW9i;ghzHD(no}VE>)!Tz(dDxLP{`AKk~$j1aoD?7ui|ndFEyOgI2G=GT~@ zU5|rD)hB3%;Pv1ho3sA3w+VyStZ!#F1K*+tda`S_o7k(x$3kK_f!|o8JbnEKe&x}fb#wGwNc1EjqtKkvPsZz#+>1`NAf;hMWtAIB06yP zF+8X4%{qz!)2WKf`rvK}c?5`7=DdvXSMQ{WLTYkk8P59)=W8RE`AJ6o{od^A9`eK< z(}S2QhfY(h$dCtcgZ00bISi@lRACAY&AEG@%~UgrD@_a2gsvxZ(A10O%3oxaJSa6o zKVgBIhA|07`n#i36=B7Mt=K0mEVi~9i1SR-d4uD+M<^Uwm zzLic@ROeD*24CI-b=f6&#A-?u+6943;M8gQvqftx%7me+DPmlep~7>xTWHBeTt8b-I)nCfGjW?ZK}h zCN;)i4Z=2IazYM{FmQM0)>I|fd1i5D*IZItaNbr~gz1tX9>mY?8@w=%@4y6Vw5!@q z$Yce<8!=J8*RdHQ^J~p2351(KH!-`%t3Fw3QZP$I_)uGpX#!V=yYNVsUU%Fm@Y*b2~k z()#apOno8IoNzp7MF-e^N|Anz{)Pt9cWq=7Le+tgJ}PTdc-ipT(2c%fND?sX3A#rH)Q&ig_%vbr1D5V!fxJWK+3k#%h(38Nc%C9)DzjAY`|Jnq2TaGRLa z$VWL}C@eVA&VCS>rRq|J!DSmUQjknsrpL{|yFdMaWFh#L@KsP*x)781kQ{Dy))Ll7&-OfFoa5Yi;#zIIHn756VLJ1Y@gIi(^H66M5D5zcI2m@gWlY1Gq zs|}jvG=X0avTg82Yio97f1}9Fn3s2tRjWQ)Jl_X{#;S%HZpEaOTIgHAn3M`A(#@YD zoaVSfs$ox=4gtkX$4;yx&yCD}z^TQa&!ry~7*;o?vZiaFw6A|?Ff&k5+7A;3A#z(N z5YqO)@rv-PlZ9v)mlf)NE0=~%`b~S-73yE`O}y!0%0St z4Vz-NJ+9~z-ck#9T|K)xn;M!vgITUW-t%;s{(AUbs{mZsR&>5gaCKa8P@@qjfG3Iv zdQO;zmiy$x`;JYZXY%{e=)yqxJ05|(aj>6xm|GeElI{ndvRM8Kbl?c#Iq1ZuGlVc; z3NkJ7OTgeb_m95ZO8Xxi{e)W&T3ydhl`orLX(k3^O{#A(+*|BDi0*t(^R&!h`jV=U z;!taj%~|3-PFEAfdjz?XJS89Bhf$CQ{z8A zJOf*c6{jI2$=At1pi|Z@kOj#18#9g7uoJI9Qf8L7`ZVOi!xQVVF6T6(=&_O(*JA3| zw^PNp;w=38wVGwr7v1ulmuT$x%XCl>%?srp{}>FQ-Tt{K)g;&+CGM3W;f<;J^y1G? z3b6+26l~Ss*q(r7HiE+$+kY6ppER;-`RX!tvXEWVhKwqg;R&|>UJFK@qhjvJv#VKz z9-XioBN573)w6X=3G6&@-4``lio%V>)qlWJoU!HKcxm0JG+Uy@65T zZAIEt!L&F#46z4Bdg|n>KsKAUMxH$+L{;nqS2n3^^E#&`3yT6yqCg9y`L`s;FP(d1 zjiKp@tiYKs#HP2(?FRiZN#B{B;XHRjsuuKQF7#t(<9#%$K1rRh|AmTYS*Veq@=5!7 zGR6jIM;{a%sxo!(KB79$MSy zo{0%RM59=+iU<2EF!z~Rp$STb?=^RVx~oL}j7=Gkv3d)t*W`UdG|b!19y)6*^p?0j zW3wW57&2|g5cvuZ-$EWj)dKH6b)*i=r_&^-E9aJB){)5bv2GRZLLa*Lq>iv7F>Ei@ zXvv`34`nSK4~-|{A(C$yIMW$kqXpj3;T9-U8@x_)ygU_{{lzeorb~1gFI7L$v6D9E z94nKpI+L=@Rp{J^GEck#<(|t^VfJ(JHqVVuYHOgyN!=9%FayfC$tb;A^jzn?_^ZEl zFsvk~3YmxWc74#w*+|jurktepCmP?F{4%o-Na2NdmEpi@+g+G$9&;X^uuvH%xBcG0Z;0c|)J~=++oeI6__+?}$ z8ViBY+LH_;C^d7nOAnkEM`P>Z$>~*)x!k`T{>V?%d->DhI3*$%49EJT5%)?w=tqs8U z3vDKLGf+s49*wKFd%8!*kd?RbA--*xleRR^cf$*bXwDw%Qztm5XxFjK;W6ATOwx_Kixoc>JMiA;(05; zGV12cRnRqf>2Qrk$VYSB0yT}%}xGPEA8Vr2$s0QqQPd?GvmbQ-yZqo{jR43h) zd?VE9P_e<;e;6m-1_?ON*(8*li}IvkGHBw=Z}wOIL@xC%#SH993RSBed!1x4o89nX zdrq?2kMI)_yWDl5UlcTAxpH0zJ1BzEJ!RMeCz(=1@g1kp-G>I%0wU6T1pXQy{a}`& z@1l7;u532THUoJ{Xf&8Zhw$8%PcTm$T0asW&8? zneSb&;>bWtP2nW^UL86Fq!6cBK7r*qHC$3{5Y zX#rm1X2Q862XRBbmnYwsyCoEkkksWedlftao8tnk(VE19K9CCNiB0;PvubLRK$@n{(r|9R^7hW3vC-tXA@W|p zi~;5VvS=vZvn$CzLlnU8(Fa!VaL?Q49I}^B`ur16b3&O5F9gBCyhDtWK zae62gk+JNml`2yTj$yG$c@R9vzu$)WciUUTgl$g=ZHy=1Ns7cg^d&52VlinUz)$_= z`V~k6pi!bdn5Z%_pq3Hl#qFJRG(nY$Th*=C{@!VH2qR^^nlCZvw)gH;d8at+e(N!Xu~mo0O`D@nrTyfv=sZ(KlWGYJ=I$uZy|` ziQ=g4e$Z{nI1sMs8(DQhqI$Rgoh?%HDRbl884=|fCX4#q^BbcZb4&BQW+J9Qa|107 zCr^>VImP`t)O}@1NQz&;m{HO#b4esfZRJ~=2}_QKx8bJ?Uozf-Yy2PyvNzA%ZT}=e zk$7)n%{yXm^MM7r9O98LI2#m9_F4Fe3?uSAI@cB+4;ZOSSYh8Sy;=* zZMRG@eb!4X8FJFlKWK{O>L7<&5(p(VxN{9gd(V?U1$b#;h)$QW+8h%Ycvl-8u z{SIpl2Ec-|IOmvWEPu3dSaQ0n^OG7qeS585Y3%YMc89}GMA*^t4Tb}*)BUIsXu!O+ zWop8ShE{`+Z$kZ**WBH^G+q6=3w3zHH3%J=8;VTIL*_MR1KKqwa21ca=^CrnToTne zC^7dqeqE|-oTi#Zdo|SPho;skJMj{*mpj_Kk5)CtdkzD82_4f*6cMweE;763y1l=v zNO&&O8&XE8v$)U=>W#6WP=Sm73xzY@yOZ_|8u)0Cu(3<}2?jM%p02AFQ~bHZEQARs zu3jCP_0Xr=F1RccR3IOwe!bez@-NVKwSt`!s^{XnRGlLo+1ps}W-XtIS(7m~)sDFk z(iX5B%@aj~*FAAerZGeh#++nGE3)v*Zjp!E&kjL#bHSg8sVXw#QaPkE167)!>*3{jh>PtH!P(aob4LayC)`89OWp$&xBnuR zmvn$h5^cH)BNex8gTYe&T@rUzRNQ96FnenzNTpT^>nV{>Z zz*GsyAZ0O!{Z52~zVk4j!+b!14K5{^KBvVxp0f{=If}r{c*DxpNTf63WB%i{U$OF- zw~zq&d=>bP3FwjUFYzC*A62lD6p~+&Uxmz{Ux|CO-{!9qZ&Eu_PYUW+<&~tKUdz~Y z)@b{SH0EWIY4q(hA(!gz=>qjj(?eonp+%oQmkD#q1IVA*2FtDkFG-nHhjP!LXOCG- zuHN~CoP;nVa^n@uV)3W0{P&urkqKeBB-1{KKz!!IAVp=WV4Y;XKllK0+nCV(l{ftr znDEL?`f9gpvewx1y;iI4r_|lr;_iJ!rC@Yn>iotF5qJatik&@DAbtc>((5jIhw>Mz zEC{$uSy4d}v2ZzTiFt*34>>8GEQmx+G;*5CBarTDEGBtE*}7|q6^KN_h<7oSXD%QE z`UJ|))53X>1UTt7$~$7(nsI6FPfF~P)@+|2Z)r`qk4)29?RFL%Rf|`d8U##H4DL`s zcm8%9e^R@5-A|y;=_X6f=kvn6&&4A7kB(cOdz4k~77Msk3wCvqklBygKB}QbkRRU& zq>D};*+lnWI0SC|Hiu~hxSOUSzP~7sjH(;TORtNx3IELZUzBW$d@4}RwY4yu0TQxu zEwPhx@dkKr*B@@*&yv#~&Gr@dW10|<6&{`wniuaDs^`+L#;vg@jtx1C4Uw4&%l)m6 zGLx86cK8IE5GfV1Yv`B4L@XAXDtNsjsIijrDd@4GIh0tdM}9&o5enV1lO{l@LO4oV z1VVSrfl*@g9fn$ZO#c@K(&vDGZ)3YO$t%SEJ7Fzk@f%e#u`NNtrKuIOu0R%m2FCEXZ%AXF2D3b)?BKb4KUTnNZux;6_7h<0$uIC&Iu_ zU-mGQ+*pX8%qwb{-hofEWDJ4CAV|5i?7b>y&!sAz2{0_u*GyU$IwrgT))|Z9#kJsC z7swPa?5z~$j>Y6N$Nw}i^oo4@0z$0QAj_Cm1NhTliOAgXki9`k6_i5HEeKxaHA3n= z6Y{6!q3g@!)dGI)OCZ*Ipr^kMpxv{5IM3Y!Up=CjyK@6QvwFu`kiKkBjE18gBdvI@ z^tER~SFPH2LN*isULPZ`m4aLUD&u4L&j#{LuQu+*Wsg5!zb>47YdAc=X7FXzS+w4C zKiK1v%pwz8Yy?jaoQSh>FyoCH zigdUo4$nVlf2O|cOz_Q09ztjIyiH_^n>{|IJ%HozIrgO0i&rZ6cKy90?I*1-dx?6P z2_S?hp_5vhn_{;1bW_728qX%JdOiQmO4zxB$!9_n_HGT$h6>ls46P@n-0rkcp*A&J z<+X_a*n88sq}OlnzhO66xh*XHH=buN6o>IOO!z+DOyYP(w278wvf24sx8A1E8dh}lcGtS912PbK$)n z-lsR6)mrXUi0#o(3(8LEL2PqH{P37MFw#r~)hU=9Q2e9p;^!FAa1@LqZT=K++au}b z|MJiMX442zHe4Ipd6quGbhY*PEaa1oERQK#kSZ*5aFhBnQdOlZgMZQbfk-!P&hBe| zQ(e@M$i6H@JO`M&vTx`r=>j2O8NmqhP;;@+lHrsU(L#`3k8S4S5~#1FGZBdWSNRwKOv+E|*KXXN&d3Q38Q=j}J-R+to=iG~ z2$0ZcSemJ!&!Y_`&an&FAbdB;JZikU;V{BoD{Lxez9$lprm%c(T(GTL{<^4HmxQL8obHTunqDt)bL zxNVe|VdR#f`ty^{+Y%9*8A+v+93s*%$$0xl{;}foaR2BZ2c8ujCk83d zc|VlV<@-{D^P9fo>8t2To1xj)g; z7wn-v4aM!j@b#2mm|1U=n~+Z4SlP`ZR>=Hqsec|^ozXmvChm3h(UYOtEpzF~SXtWm zv7Q3WksgKkC&3$Ubh_?;rgI=W5wr^w-4Hq$`&6*)%Z=iEamHf5ug^jI?3zSrTCf(W zBvNA^f`xm@=iq|{5N&}A%iC4^lH?$x7oPnrnx(bpZ`Row)wQcyb!kKcledf6V;Rn6 zji`vUZoE6`qdDR|8v4l`abzBz@?Gr|UH^}i%HDqL%v@zLV}f8dcYHYUE5rpVJ-6%{ zt@yw8xnG}q`qx*rG-u^KGMdW2NMB%u?JB6VH9sa}-J@Yb5+`8J?IH~Z2JGv9Tm)cu zhpO+RKV@`-h%*a!It8>}OMKmzr z<3Y%hzgJ~Vojgb+TvvbmSCo7t4|m3Gc;<>WalZ3d_lTN5|47~P~=A#W+H{MqxL)>gMv{O73d90^HRhGs?NT0#sT3d#x{Dok9T2xC_F+W@0i z_cryTpJy^YgL%a(IpZ`{7!#d5w|hDR<*=a5C}W`t<%oy2A8{KuAjX0_g> zwHT#!pLMy>@M%v2TQ;@&1*ehNQ-5UdPu@G8F4As)REk>MtG^%jmfC7>!0YcBZCm)pf>;WfysG$}RHm611p?eTQv{T$>&l=wlI z_~D~l=aU{={kn?SK9WZ9|9qrl<4`@TNH_eKkpJJW^1nS7|Ie(-hyuPbQDmVevTzg^`2H@WzKueaB=uxO%WbqEK`%pF6kLxRkP8o&}m zwV)rvkRXT|>Tf_4R;uKruke-@rkTl4zyW;)o{1A2Gw{Y=d?>~L5@+Hl26*;!r61m_#bB*=#=CN{vnm?-Bp8c!OtLT1Ql8*&)pO zcdoC6ik4QFPu?pNbii1tdF;X54%NZoX(KOtaj@FZ!Q1^+pF$7b?BYI9#lU*TJtId= zecp?@W%J(fjkP_#iJI}j_>kWXa|{`?hDx>AI{$|VeSlb=oud$#gQgD8rflz;4jt4} z0#@OREqV3dHR>91(9X}63m;-eZB{h2b=Dh$^zB$k&@>iC>V%nN#oNG09ZSL09F&tFf zmLD@sf4x}cWLjw&w7C#WY4)te|EadJd0!&c*EezQa-Ao-SxM;~4nhOw@|-OT%W=Qz#YSk$>TDG7@VD?JuS)Gp@N$Gc$EZ z=p&2vi@Jn7b{D}>m;{zb*|WScpBg&byneUE92$3V^ts<*A?P^6cuF}u9+AiXlSPE6 zcb)iLy_3ZAA=tiB)Q+*!NlDiDG;1;H|LQT@B&=|Fx!*+1?peL!Eo|8Huy`D`!wlDdl#oIFs{9?5fcZGIbkC2`hVjZ_1rW(G?+Rvd zC^Q}=BfW<#e@L{U8^4(9^Y>+Rxh?NRDd3L_3@A;$KEXjP69o4&uuTk`gjn!vYL?)K z!4qAc#)9kq#SZeR-mt^{R9p44t%$a8%7g26JAx{!^@I)b`zs@Fp28ACzLVcpxHG&d zhLz5J(=98yndj$OooSvXg8orasA{C1&v;ldDV1~fKD}yFOO%c<&@*Z$S(R~cxe#|5 zCg8~ng}!HlNqUk}sQY}VX$fqbj=EeEcT09r;K*ucC_98`l?m&1ERBOq9_UY60_p~W-UG^`2X-jd zs6$@XlcP&{2A~c|)8ggC;+Vki{^?0@6BP#A?YN31)hn=>q#jqX-UPLb-g)v>XaX@ zFR3qUZOxSC0@Z~l5Op4g1Xud-sjc{aLQI6oA{~W zz5*Y5AoO@jgBrD!ZV;71EyTaT?~Q7_=$o#7RMe%4{zcCiQbFo>hih5BKA!nqTa);nT?QQ7eZ zCo{L&4y^vyBq^ZCJ|T_w(P#dRa&ja0&6E3&f&_^igVmiR)z{>6-2ZThuWlRVRw-vG z28}J{#f(?Rx2zwJs6JG>v2sF{JEL@5SP1vGj_0i}$^MosTZC3Mg|Qx}UH=;DSw>XU*5)3W+Y3(3*~(V_c) z_@<(zWFWe=a<;`lpYtCKz{)i!oXDPZS}cF8&&Hj4pTc@ktezap1tx1%RNu5MI=jGO z)Lzh6yZAx8t3e^6DdaCOkYZkM%%FkmI$6LJ{RO*VS}G|Mr7ibYhq=1laP1@+9Z}$g zYkYFFT2SKX``In`lViIbmg-|>3)@S7CaNYwBd_sx>IWFc#_PG7nj~)MLF|hYZ>;@+ zSn;^o5q3@J>&bdxn$(TN!3u==MHou0YRcTQl%2=BSFdK!>Y`T~1#ZsLHE_M*8d#e# zUZsFfZ0B4t)@{oK95IhHV%4L1OmojMNJlv;%tNKLpt9VAYYPqOxCOeRI8wNieaL#H z441G3gP0Ue;J#@yKP9(%CFEjfsO^#m;*ub9(V$PKzhJR_8bV-E<4&ZSjNe4l!Mo87 zcK+%3lMxZ{UMM(sdJ@$gzW*z5q`}3X*4~Xre9sz~Em!0@xqvEi+@GJ&WaKMo`p*)@6{-w57e}z~&pmEO^J9?9lmRNceZI+Q(*+f} zqgRCQ3zAFJQN5s-))K(a8RwYF04cPOb|xwG?N5vAaCE+Dzk_PwvNOT(WhG%+bpDl8 zAoiN6-%dD9PN@vYH8qmk>K=oOhR;@FKq}f{E`#SRgI$u|7!W3WI@;XGK9A$>pRB>1 z`P?Yi7(2yEfZ0jh$X2H-B2m99LQ}&wwD5OKa|)7t3iAvI)+)4a$`7moQs?@zD&>eM z{YgcTk$D*@#B^*SW^psvBgs-c^emZYklbgCzf$h7ZGxMu)57d?;3CR`%b+KLW;9el za%A^jf&oE+vK7~&Z!3+j2i8QldL*7YDkgu_I)M!)xKxbsVs5!h2q?Rgl&D8&1cTBA=VAF#{FB8v}ZzyBQj^Ug! zs;dbl*1OIRt1|rMIitBoVn0A(0lBi4a zWM@r2C?2fjb$}_T_Z%4}y6=Mp^^GR#F9!RaCG-=!u)XI4{sQ38{sQ0#M~3@h^!iPA>^Th35-o8DDi!Se4tw z%@+FB{(Sf|S7@}KQO-;cy}CfPu86cP@=bE~P)FOsMNlqmPsY>}UWr^+_*=0H=c2l9 z0(LO{s?+chS{&*rGlY^_;)*iYVBZ`h4B2IY4E=kbtff85Lli|lg}L?`*bl{KGIM50 z`7;bY{o#L2bc0+d%~Q1y-KydCakr z1kjES>^r-epk*D2D*G#ZLgJ3pUf8-2;R!7yH5-+G(Pp0Z(Eh@VTgexC;Di5U=$Cm8 zQ2Dt9fM;NS8=x-yeV-Qe?NrJ-gj~geg)yQt*}LpeH@{LnoxH0gn_<^NaLQ(fZ#|cA za|`yp-lX6c4<`v8*AMA&+TNrWbFz^$BtD(^ZUn<#h^2kL30XxVY&^^|9HA$qRmmJf zokW)OhIQ>jxcrgE$_`Lc%2_`W5z^+SuJo+6@pj_v9fNdYu~}_tHe*E@oCf&Kn4ysoeB#3tbwAQ{WX_|~o88%_;a+rRi+qU$ol0N?oBVXK* zn7r;}*cr4iN?F7>I-GCwzMb4YOtC|_BUQ^+eq=>$^9x!=bmk2b-o42-ZatyDldpey ztKV=qe6i?YWq3xM0Ml@1B~J>g;r;`w5ugUM_UpEYyz;mK%-sesAs=pQt@}oF`1rPl%gC5%?M%XP^*E?<+e_}T>2}rfcHNxe zV|MHCl3R&@hg)d?K!v+)c4Z%-xi9MCa1A)qDMINx%*ae#N*40xY;1LCzIS(s7nfkb z%B%r4JGn3gZ4|2Y4Dm!n;8i{PcOy=dRWRowr8dV^u z<8AEu3;zL1MEa?bsVqS~DBh^Q8+>$19Yw1QIkkOSR!pTGj~Y*a^0M5f%T*31SrozF z1hI*wM^a|wBtOaOaKnbH%mSgNDlBf*#}`SXF8DM^XG`SP39r-6>NG_>?=`n`UwH{8 zd5k~Qcw2T5eM*3@CwV>ebUBg6{_TqE?clnKyym(uC6obwMv_(F)xN}7d~yLY5+-H! z#V-Vhcv@0Bj#MuF!AJ>{y#Xl^JgW>n1rPIOOcP!(x49Iy%2pogn5eD%v_qtiis%0) zK4vStWoctx{WL@1HHbA10fZv6gsJUAXJ*0$-K8`wMv++p%HS{ikdEEhW6;*C>i2Al zfykxX6Z+D(b;A!ZQe|m}(LF~A=|)eFyFP(XfyMYd-tC4D zJlkzLe_0FkGHNp|z-A2LPExfv?F;ccCMp2mL>A#%!9>Dl>=%^#x5Zwr;r8UQMH_Y3 zgUfDMKmqPzf|IL_wZ@7v#F5@6Xps;l{eD5<5rA8YZA?8%wM$K1 z#jHt`0=HZG)b~mlz|i7IW1lF08i9UwVc_2Gkdta9+$l>)_hz_Gb83Rst~YW04TKfK z;@>ox!vxy)sWjgCT(1MUWP85NiG$PF)!xiBN5-1x5F_E5N5&t;%sc(K`MCeF%9r@* z;9z5vy}dU#d`AD}t?Yi^CtE2Y*sy1Hp>BOd{4ay@HPRihQctiBrCce0bHet98G||v zm+Rv-j$77FmLqtdJf`Wp7`7@G;n#M+`vT^ju81ZFL^O+B$F0t0m3t$Oz#xQXl0!2J zf9Ue0>*7HuREu$J;Tb&zNk6%i`PFt8N#K&pcj^vT*@>hyjO+E#kFc`4hMp&%!^DK$ zj0!l!KE&^R5wb7wZtp*E7fNe*Umv#1BPr$z^TzjXLgxO0;p!w)HKfv1B0=%wLD$q} zI>8gHMY%?B^mi5pzW$j~0i71-R~!(Y>87fTl_|?T+Xamm#Dn5PG&d@CAa|+MEP&oj z+6Chdl~h6tx3hEjK)yA5tl^GHP?nbVW*Q^lWzr{52&W5K!D>G@!n&u1%K_vC6_Z{R zkJC@hQELdMjCVr&I#{_O(a4>2QGd|FGnqdU&e48ST~bnrFHd)^4BphT)B*W6)Q2tB zI&mj3+{N-rWY^Lt%3h#VAc|1M6eyWtl8TW@^}Sd!xklDIawKA`>lRY5^4=!u<_}P7 zdY3^6dIV7RF5pLXZXG&FV$ZUHeUWSeHJ>dc70gb^O@DttBR@=Pjqcu`fIO)rbU?Mg zKyA?w-4IDaAJl{IM5%~W8CJtl#n4MRcC-uA)>695YdZ>-?*yN*8h#s61G(Sc%EQj+ zcXfhC7yWg6eZK4Y&jc(=rq z*D7r+=Uw2zwez2rFgRdyf6>P9U05^V&VaOtjgQVLrlfa`-V8C+_|&a{@c5qRr4PQF z%_>V)J}gUDng+g{h_r$72KW((s{H&5nf4TCTe;{;-PT`Y&DIDPXj$vwYAR z`hba4#{_dodAb#-rIF~a+3~}R`qAR~t=ZI50CM>hhUoDz&nx6{8)Ov2$$7w21$fQ2 zytPQ56z^VM3W2WU?l9q_ZcBJxJ;H0@NmZOe5BVc?&vmC0xvt8a2Cp1qT4&?P^Py3jABB;maqp`L%t9Uh!dJzfbzqHKgUgX7xJ2 zL(yX<#S9*@54_Td>=q#lxOJ+-Z2RgzIj*-3sUeQQV*W!v_h(#FYtNE*osN#%%xPeF zT1@k)MThw)5~7+-8|;oyvx+~n=4V9Xa^wv7<*@hcT${X-AW~vp>hrVvNACT)aB4oC z|Ahk_HXVmFWzT9!MR@s%Cq*WEWjMaQC5B|5swzWD`Yy`kZs7bj72v>kp6C?SGF)#h zhmFIb$xO6F=)Q*`PqmASICMYIwn)U9#u;jzw$td1k<9RU2v=;`krGPux?qkh-e?$2 ze*0{4`R<~rC==EZ(g|!JcK2>;;cqDBm8MowGg@Alcn`&QvUnkyV5@K-@=#%AJSWtv zp?`_Cm>pbBz>}6DM7Et3EC3!ZNu)M6Its)CCUV8-MDUu_Vkw&bkAHN-|5^H^e&83T zzv4^yCGu|TJmfuwwp+(UKW3E$)_1@N=8Zmx&ppQ23LW{>2LiKA?D%Qi>3}2?a2Dl$>J;!AH%r6syZ>0R@weQnnMoPu$ zPO6bfzK>~vvA?tB7XR#6eo7P)x%{ zX*qjR4ZYaQ@e6f7b4=^r?2?I3Qq$uGcUMG#G-XQ`b2ibg^Fus0{Y%Z^E)eg`?ax&s z;GoX_#I6?ul~LOw5#jZh4;=j3%4zA;MD#V;oD^W9SufHxSW@+BP-}q3mx7?q6d6SI zd%==sokRZ}brre&wxfeBp-jQ7$sOzmKj$9At57u%-i7puqt0A*N%>hbuF^+2%ywu= zds@mDP!?rl@!cs})AWc1FK*kXxp9IJokI6VK*=jTMkn?bik%9XZ<#K2v(G1 z1xE-$bW3dBYtqYPsa@Htep5~icGC!YE4pH>x;OXForJ-6&r1Z7{eA~xpPJ0MsUh0B zk6+-(o`w!>@Mt~E_Ui0?b$;2f)ihGQlCxPj7<4n6;F*+W8J3%Tg5}u9%kDE-Nqzov zCaOi+Uhqti-B!H*=-ihO&ws7-%xk#sZ+$Bc5PDLEIv9-wiT~VfQZ}R-pl5qEC;NlQZ;FxvUcTMo z)oC~TrRnWDjPf=hs{^@XVRvWUira#uTU*Y|$3@1(wrsme+BN!^$oJV}NS8T@v+HW5 zdrCC1Cr4CWVKd7{E+Ony`z5|g^X7ApRmzcM8{&2Z+9mjgZhAJu09py=gqCiQEL{j1ZzjKZt_n&1hir7p^b z_n7G$?*uv@Q53OmhjNv;0< zFJ1p(=#_yW+YE)V79uT9-t6wwhiKC-i;Pl3$svZDi_HDuq9P9qL%jPCWL_NmTU|&v zA?N!-&h!u_;$|gk9Z&|i-3)tN^DCYMhG%+=-l6U^f;JsNK4ChU~>&aU# zmkCu!T;QgM$C5nvB%4Ul+W(7IS-lsx8Wq*BwhT~=>t;#})7YGkD>gSJj+F}}Z;!8? zE4?mO^%9WTY~@(B8%0`z{k~$#m#m~pEZX;*fPtTtdy!i|lEK|8Zim13G$1~xxzp+PN^-*Fm5{Sdvkcq{nE$Kr+Cqw@w}Hs%$gz;%NJiL^;(Zdi zuS@8xh9hc?;UEIPEc_reL{II3tWn?toZAXrAg#FeFFf27<=ZEi8R)|bhY1X(p`LK| z|Ov3;|@)Z zRGGbdHGbOKiB4ar95+{{W-W3DL^it}bH$Wp{$YRtgE1s2+QxhqK{}?kY-G<2<^I{6 zuP*oLU5e~1(ztS0iehU4Yxp}s)LzE!uYO0O%wGgYqF?t~%(W=XCM;k+spm!$Xqx2n zW>3P0@*_VE0oJVf)A-Y>Q!K%sjp5_@&kaA0+YDXm)EuZXGK7EWX9n};9WS(e32oiT z%a=5bzfPOk!H~x`2aJP$iPcZx8mhhfnH3+l5$awjCSQLOh@YcC$5z_4Nb3}9txCj0?On1>hpv{y* zeV;u71~A*;cA-xiT#GUQ2BdAjij%!|o~mELyCdo9VMX5?O2|FOHJQ}=;T~>SB#3b% zISsy9XOK23E~-MoBExU~2%)Fm>;4I^>g}t2PrZ;{*tO{GdKRIf)`Ilzy+ACU)jEPp z*d#{e5>@Q0!qm_wAa3e(=kSs=ZutWna0U1f@-6gwseV(Y%A3Huht%>xN^HO-?rS){ z;U_YsLMC)HtCYKou09p@W9e9yU1kfluzUEqV0{N_kG4%V8TF3$?Omc#0JCh&Ouyk& z1+{JPu;21A2Yywk8&H0#ds%Al#F*77_==zZB-_*@cy|WQWr049w|)&x0KCtaR1{QK zn^ot2FmzlWoN|;1H#CAm0HE!yWC;DxR3CEvUS{5e17-#;QoLF&GP?p{tFQAt9Qs%c+ga0>c!uF&S#LI& zGt{0!6NcFW;sQ1UUN*LM&SvVaMKf>F+ZRB0H=K`bO&mKuUiZw1M#E~=3BUg!jFP@> zf}1HZmER||E}HH|E~T^YpE*0$bx+OCPxL%lfg4`hnq=r1Re9OY3Yl}^3ggoe1ggjF z5~x?`yO+Op5q^!CKZ#4=81{PLGBPBVrZ z&f`PAzNUU}I7Us}Wa@zt`4ErO?K@D@Uj8QNES0Yo9Z{7~TOOuJ2Im zT5#&T>80W-X%Th@1q~>>_i4z6&Y6$7k-D!JCUuQZ+=$F)v%FN-SK?bSm4qMM2&S{S z;F|AyFU?nC`6+rLq(4Vo-CzleYPd{|<}QqyaD9_?6(|P$6gFK^Kl3_Kb$ER+H0tf> zIS=A?b*?avun7kHGHpK$h2-B*kAQ3T`Z+Y+JSS`gq?rI8FEr1EKxlZ*a<%H`C&BM( z3W~c?nRX*QVboI{Y9ax0YSXU+a~A7=Y2yPf0(p4JUib&H4^Lgza!WFnR5PH!;|!oQJ({#ik?(wa51TUK?zXG9(t$!o1G-K%AIkJVaREF)Xo`mtEt zFBpf`2RjOCcc)x<#ojUjeqz%o3&`MUvZIid^)ZCA*)x>t239=$B$?3I7ms3~j*#^( z0bD12zJuW2kn;@uFiT@(!)S1OM}pKX8Z4>D$0_4uZg;`hlDibel2-QNT0(PzW?!j( zpkmZc*5nEBq&9<4XSP$CFzkCu0}6BxG6wQ9*k*ghl%xf;Id^O=R$H_?A$2_Ih4bV`VS7pn-|0p?=fT7J-&H z{n8fj+0aVvxA3)V?s3+yZY^97j2HoyBSL}$&YXqn-wHu{NBZ%8es&|)br4mc7=rdI z_#wmdR<6^&n0P@Z;JX#y6ap)^q06H|!tP}^Ve^u8Qmf6krzU}sxt=q`oalG(Am10i z3MU2O9q^z|-ZR8*hC0lt&svU& zULBa(um4I2lh#fS*IV3(96}_{;-sZo&I3EQx5Eb}LrCItMvqbjaXVerKUaqRzS*lq zP8=_o25VDvaKqyy)9?l&SPg0H<|ss(Q`Fb2iBejfsD9vxj9uVHMGV^SGi>nD%~uV( ztLn{s?13~q<`Fk!DPwRsVEa_OjV(GQl+eu3NC@p7|L`742?T-T1uX~$!PJ3o1L0=z zp&Q^UmCW^LP=*H;)C)ER52_Ieb9Md^r2MZgdg;P znjDfwcfh;IGaL!7q_W9H?u-0fm*^08*-oq8H1o7Vhr zH!HeG3Dw4qnGPT@B=Wslx;dP)P{$-GR%{+$d3u3XffE+Je>JuHhN>~m8S@R*h0lt8 zCIwlYA$`*-6YsS88atMrhh!`h`!A`}f}Sc%8# zJ$(!*Gr>sxqou{txd6-3NI<`5&-XdaAcRS!F}>^#m~lk|l<=M#H@&_!L`ri@94?S# zZQfqWAFG_iI2Ng=9fCP``uan{zxM8t9t|R}k`V~FG4+`=r%y(CGY+9D^Er)*Y;+Lp z`vgI~Qy;q(`o1k?SG4eO&L%HRHMTu$F0{8x;3;-qyt}XB4MnA$32#c|rDmI`pZDwububd9m?qK^yHpJ2Ou?W9q%2u8PRhP-0F$D$#}XlJ~T;EYOGQO&AA? zt1t87CrOj6?$iVWE8ie6gkHjQpmycsva2p)$C#ZB`#@VUEy153O#ml9-$HgYbO$aj zrTN;s11_-N@i?C==|eSObJ3Ta@PpI?_1_R#%broJpWnjU@kaZYd%Ll1F*W(f7_-!{ zWha5K-4Fdll)H@()u;S|MG3eW372N|uh#jx76E?*&wQRYPNBz%mADg2;jeO$jocoW zBI?zw2~Z`{b=i@!hqyN&?V>I!Z8Ou^N7^eguyu9We5nFr{k?MoCaH1ji2LA8W)(|v zr7x)q(@peW42Wy{TB@VZDQ5qZ&et}OM|gaD=@oK4(xf|BA@2xioMUz2XrzW7`|!4f z*H~aq?TF+SW8vtUHSa&Iwu9teM{fxBd1AkZzL_*#k?G=iBrQ7 z#RALl^_~g1!E>AL9grr-C&0U&-)%(Bwgf$m)a2S-f37UJWm5Il%qT zjyn?+^V^zx4XdwI{L>!u5w*Im)?bOce?FzFOnv;y2>w5)rsy6E!H zkNo>pR+EtWpRbCwon`oc(|_|c_*@cqG4D7GaZK8K6D3|;YHR}!lTh23GGlBSi5V+e zTA(rP*&czMVO+A=b8!Iqa(O^)D}{U}#fEo%o}sS(+3u0D0_XF@@eZ>_>iNM-)PSHF z?a6&;#e`sAJ$C z>SyKU`b1X<0Y>HT0<+Yl(%|zcy4#(3%|$>9i&zSm=Yam+`QxlneM+?TaSa*nVGzA-Y+8)fC#K> z3+J(2!nw!EuYXbkDBM#CnW8~^Y)||`XL-lMr=8^-2q!n1i}m6a=6rVLENNXbDcm&b z)2R+rql`Nlgs@*y$sJ}Kvth48QI*k%wD6Kd$HYRdrP&KeU^EJxxbRjwz&@t4Vbuh^!IdI7jmD<6O>ud&oegIo6P&G zOkLT_=MdWilt!4`|XrdcfO zK%nfRY@(RK{6XyY`GBMJ`=c`|PUTGhhTS==^Cv7PM&iQOMklgksBg0iVF_X_#;Ez@ zC)nWo!>--LFdDr;(9zg*F)%!d-+EpRdk%-MRz_0urEwI7&^>zwV*sWbPar^CDXbbKK$8MKqG zyazsD6P#gHQaD}B<+NuK?XxmPu5a4|{)ig)hjV9aPNIK+-B&a)m5>C8VQUY*{`$j( zx-Y?3_xCv^Ain0$SImHfZ;a>!<$v0opZwQIJoCECFe1 zYm~oJPW`1`B>1oyc3Cylg=P5l$~Ij;hl$MT%t(iTsNCUDj&&|X>&Q!RO<*~@!*R)N zr2Jw_ba4OO_xr}p$EkZ-AILsLo0YA^PJ=`y48k!7p9)jSGH%dpV}hKInQki+Z%45^ zVm{p0<^Zq(ln1GrntTZEtRej#<}kF^eGhibSib=}U#SeR7DX&~8YTGX+?@e z)zB9PvjgQptApRbhDKBH#n*#i(2lk;>iILK25H9#iAQZceM*gcG4M<<6%tuv(<$)ZV$G)(RlEW*z95u+NekQAjd{P_4R-RT%Llu#k z@2gFN42XJOV&>9*LE-!nq~nl9DuY0uhC+RUO_HqonMemO$_{wgEcQE?$V{b7vK$7q zTg0|Y?L&4g!F3MZt$0Iz5dFwwT0FiG3fP*2A^<~z=8!u#!{dURon*v;Jc#b2nQ9n1 z(74|$*|hn-Dd~7alsF2;!016`ok*BS%X+wJ(r90wU$VU`U5!&NZ>L`{*=O6#YOWd! zUaVFu+@R_3LawVCk%M7&2>N50O9f~^EU)h&=yuDs{3^A6uQxrx^J7u<9dAA^dh9V2 zwjp<;eHn^(RB?lBV@8`5zw#~5Y5{0Wt6y45=a32c!7|vGX;6zCJ7CY&P*Gz2`pG(h zxQwHkz~)Nwm-mkLsZs*Qs+b6UMb7{QeJ!ZCK>R~%nmE}Pvg?jQL0HcuT4!CQ4)6IR;sKMD<`$JwW37>a! zO48qxKg4q1R8TN);)NO&>;`j89CC|s82vr7k^J(eB}+>7K1RM8=^>8OT52PdciqLz z7bj6M=z{v4jL0ZM57(A@cC=FlE3=}&oAhvwy@vg})-Y4Ry&|WO^MjqgW8PY&0^Reu#1qcX#^3l09yCE={9X0ZW07 zeRd-YGT~Z`tJyp5y2Ypma1FL|%%Gyzy2eSj>*N-rM(-PW&)zLJl_(dB?=L#56&96; z17*9B6Xes~$6H@^%H`K@SkY>%bm6rgABPNrgJxW6dl$E?=+L2*O!~738*dn{htTQj z_aiZ&4pmp8ns#7mFVT$XO&&?OTT9*0}`Hhpa~+CTR= z5ua2V+{pk?6O7RobfgwZ3YtA>r6(W$oC7o7G&M~$+c)#{H#C*RGnuO8(l-nejxrRI zZ(=e9)Fx#Wb{4inZbH=PGjJ2#%!Q1JQEQnv3z*v+O(2?C_dl|RS}DqD1HJ0cx;o4K zGV!0A#~(++z1Jdh*$HkMc3)zAgD$%;JamdM5*L_u3^{zkskJWK) z0@I??lwjz|<~wMvAzb+dW`A3y|C>+J!(qSDSB1Y12S=SEfUDFRUTj^U=lfJZk~rdN z=+F3>{r>qg@AoUsuFt~~JLVqVbmo<1OOFNhemfz8;M4!ex*y_fv7`&T^&SE3ds|Q3 zYnU{WT-tGL!M@8j(KC2Gx$AHgs83i}q(Il=^!-srve3=Gk zGhF9lG&&EIlO(9aUt@6m96j-7RMOU901lEFq$@ol)Ne2E?_4QCC6CVUibJUg-R9#9 zm`kCrRo|#Qw0BI#UnY$JEsM0waCFnsTy}J6s9x{w$i(17^T^LhZz}w1sXm~#f>XYB zeWr{z?6y8$4G>!E_ZJyVe2DCk0~F_9LT#m;4DnImOQiMyrLil?j!mtd;#bN6gPi+( zxYkWqSG_K^wAc9=(5^7-)(`lpXLQU=s^DO)#SEe=jWAdtK8UyTq-+GdF+H|%;~lrY zs?HZKn_VPv@M*3i10a4`f&8@%0HTPl5S&;hsNoBj!yODS?hx-dQ`8b-L$4sC6dVtO zR2q0kBU*xsC^>jmD1=7Nq~Vr&ph_Bq#Kr{2YQihQ^l&7%tyKeN#})A^{@*JVC*fbddSvRDOcF7ig_mP{{P9cB#3$e;zhua68|q62uL!OP`? z^t)e)ArN67*{9Q%zP$uuhMP;*S~(0aN*Z_aSNS428hpu@_;^n(&f;~#r!t8{QN6ZM znMDwg2%r23DnU74PTb~3c?g4Zx+b$Khy9%rXibGPc8RpzpF73cz4lnw|A)Od4@+|U z|9+d>W#-;mS(&;khX$3*)EsHDGN;UPR>~n$Kr&Nt0Cr`WnOa&}IZ&BeIZrqOWez!F z4xpftlAZ`{C}s5GLL zwn=UplUPa9yK0}AY2=7p=A-U=2El^12jM*XhCw>&PxTm%dGxs=La^~>!gHYvkY*b{ zn5Bd%&?LohaI4h+O3oQ)j`X#4(E8SHDa9zd-xe286w4V7FJRI=E$N;^p8GtKQ^;{z7$#4zTKRBN6!{Gs}uu++o; zEde`#q8E96Kx&qiMOVn`%Cw8ANCgrcr4EfP=xB=OpL*+n_7`i3J76yLk(GMqqo12$ zsd`bbJ+G8P7mTS@hXyrGWxP#&JbUWwjz8dzM_jRCOMvxX0wiZFn_y@9IS<7EQpu?l zKYrbd*pW)FuKx7SPkVXUv}fylpn{p!`fxDRhxZpbj25x5Un znmk!TMc}cO?}CDtZaJ{7=-nEcS`=|-ONyUMZ=D$^A3aBESZj*80m3}nJdf9?(bu{m zHt72ln73L1v?b)%nP?SzhG@I;ORg*kN+j&<1*4Oa_|Cl%(BGQY(<4dzmIxP%kgwH< z*0S_it=+Gj(IPq6%hf25)O2jj{2}#lWPZ?P#?xW{9%0YduK6|$WA@ZD5=CH7oc^#E z=zpQ@jtRHu^p}tu&(GrxP%c@RsRQ4UtnLC4n1~YdT(mG~u}PXp#3GFcx}ld?*x45t zf+S3iDdF%JLbXO~{OAUd?xjKRaM$bw&1rKX%|0c76h32tTrl+})(1!MKEF=ap9M#0 zrQV5hDxfzo&$3+}4DE+K50Cd_5bYvEXhnIB3nl_vGW1${$X4kA5%l;7t$Fn*x>d{z zpZy$&`j0Mu=I9{kUX}3+%h#BueHNn3`5>H0yb{dygGTd{yO2#QmG&;)M!$-MelLRe z)&Wf+zKk}>R4REx?|q{2MEy0khL)TDdsC%6qY<3i7l zHrg}k7hx%7k5%0Y_{xRfVf${K`U7G?xps@SzUB7rpR{0^Yj&vY8QJ1|L3_vQoFlF#$w^ZM>#|Kk!K$~m;Y=vXcm8f) z`gx|f0>FRki)u&RX05gd@70IbX(@klw=dA&?2LM77@CrOC6LRGFE2>dc&87+O9%OM zNmhLi#C! zmv}=|24mL6HovJl&KOd9DvP<`JAgU6C3G{3>B%;@aND%Nr=>sfNSSYq-InVJ)HB_k z`Dqp{e23U@blBA}lFNd>aTua+)qjT;1AzGvm71Z(3R``6DN7|;sHS9m=L}f^-$q?{ z-f@n#Q_r=fjdMQx>6*arNKpTrEz!Z1xe{`fAZ6Y|cCf=){pvGIfy*Qj5PeFN z7W0+bxW+es2D$e)UR`vILgi9)7Qh1`513~j&!-)M)U1Cz&pAS#aKOa1z9D@|{pR+y z2Zm2JeI3iUzTy-py^#Cm5HY@q={7#yEbjT8L=(=Y@0?&Vy{gtTC1;yO?l6oEZSy}q z%DZn{1#r9NzLuM5db8jAS}Oc?T>YNwyLLUd9Gu6?E;kAI=hdw-7PZ~<3?TOSTgeV0 zfO!FEhU*gap2-I|t0k~_){{*y6IXQl1U_sI?^$7vj4HAxUAJl4iAy-O$HIKTvv#m& z$_5X#*r>gtZ`gSDNrZt6wETDq*~rY86i)vHJb=K_baflY|8k4CEG^RKh~jjm$}mb< z38R%8^$#&bE`ZH7mkLhH4t@bzaeAqTNRh0BB=_5Pdn<=kmS2H^%nKGDL`xFl@>JE; zbhGG)Bo!u6Grn{No#CEAc+a?w0*y zJIKitc|uT4!V%Yq^v|T%qdfO@;ZFL>E~LjJ z9F-w_9G+^D9+^B52p%BqlxB#(`2CrFD^CQy#aT=hJ>rDjQ>mG*=kZI~|9GoUQ{Eo- zuaG=S!TEZI2)6U)J_D`^wmZRcFScy}T?NvP`;{xcca3OHkUnW`omOBq$g*tc!9GOo zg2iu@in;FnmjSuP9^`WpMp!QDy;vB*<7*+zQ zuKZ}{m@}XRC)e10VB~gnn67SNrqxZjd0?ZmP$_R3#eCZTH2?oSjjUKE@Q(m}L=RK0 z8%C?b`$rH1e|{XP1cG4<{%rj)GtquzG@SYbVv1S&eMY=Atp7#K2%idFO=~O?2W`z4L+7+3sakU zvBehITfrq}L~`JpAf@z{lF}SiczxL1pzr8A4HaLw5B}b0s6%M_)-XeQgb(fDAl)-f zU`$1WltS|n|7qp?`ynoae=P?9cG2pR^&f$j6s<)Np%EaVf0W4F@1T>HM)-#nx|U4) z0V5{m9his+F-1a3QFO%I^^`Q~{rsB#t-p3|2nd9vAarB(Fiq4`iRajx;v)8sgS9bc zke=zINO{>+^$zOt5Dfm#VPEZD{`V8y-+z9|*VDgvj%Fk4e(qQc{ZO)G9p?GhZE~Nw zb!BM{sT$c6eQ(NMr>Ul2W-OU{)B^lF^|EmsC4!N<>~*U$6pdh~h+dd84JNtzoiR?zt|vAZeFXiA!r8*wMXrRH1~qRp$&+# z*mu?fraSg|BvW;b(DXibL->R_viR>i{!8@Xm&*FUBq0z!+qF%$>Ba@>7kVV6G{BnB z!p*Q!R~>3=3K5W)6NVO&uq5TbZ#n-L)r5$|NT%A;CUES|_LMah8HqE_dM_d?kCl!C zIQ33_9bV0p1&#Qvpi=s0SpDh^kBU@=V$H8~mE-z6Q&uF$$%q3>0gJqowW&vw7>)7b?pHJb}evnarb7b;Y4*o@C$i8L$1hKQc{q7sWFm3kd2dxlB{ZdyApa>(WMJTwk zQ|97EILxPnAgzUah>t(}sQ(K8wPPOu2YvLxO!kc&%uYbx{{4FC`L90dUwzX5D`EbB zLyg#FC#&?2pz0t0cn&er^COQjQT@Nfk*-^RRVe!R^tIw&J>@U*IDWqE|4Kvu|B;3Q zugG%}u!{#10K0{gEz8PmbPWo%h%nZRkSxw}8O#eKsL!}1F_*_>05&)nhOM+oy5qDN zPqFEscoDU*%&-0!e==fmVOGNJyU&J4R?vFMOwqNL_=0lCQXc7()tdlQY7*F|v!o$_x=)yxfI`+gspYUQ=OO3teEY7n+M;wMFsI|8dP1i%!?V=B+aGthE} zCS+j|gHh3o zW$GoAOG+wB$BcI4v4Q_W8@J|eaL@F;wSK-fMJspc@8cM-lh02B>a|HQL?_m#zxN{i zxisCL_C%WrBa6qI6Hvcf2u@?RW-0TQQfb^ckb9}bqw*_s!D9rcU?)1#Z zLKw%7>>>Sii&Foj$Yi15+1}Cgy}D9wO;0w7wd%f-XH$45@nfqq%K+kHcus86m(hqc zf8sFK5#bxBcGIS zm^&9WS5Rhj&GuovC#)Ljlcdq$x-$`Fzcy(q;FcFHdnSDDZmwM%X2|tUra+bLx|nw0 z{CM>>7vcU!>d`4AJI+z+?ZOT;B8gF`5o<&NiL256J{ZNGkhTIV4^e3TTBCbd?3~u+ zsi#GXiY_ZNpFh*zKx^nPc<&<~dMPs3JJt3EdoiL>lNyt zIY(>^_uGk()dTScap-M?Lyf+V`{PLbz+-*5EH)C}c?y0a5-N{kxI$hTqBCrYZO3R6 z;kjTpY`nViI4g1KVUihYNauI}{7-k#X!Ltd zq41JPojw&0EgXn?7!7+h*@m-XSV(&K`D?39o**u`l)xo;uP)ako|mXd^dF70)D!A* z78iUhYTUpe&O?so9v@k+3Ml$3s$kAA7XR^Ohqu9RYqgl~*&&=Fabje4Ev*O`&LCu#4_R-ZJ_4kl7tSbo$2+nu$9oin9%ml6awLo)G9eQ3r z0;nG46PGn-D-r?CSf(B;aBj4I_;C;cxE`66+;k%W-Rd_4h)r>_pH(m*>6(05w|4EBU z+1YT$46`%O=YHA<)Z~r-JG4@ad=&bjob+3oTe{ruQo=tkS1LZFue;g z0a`n9@#)fcNl%Hm-bXOd=%bAI$&#r%*DK(O_GQfe^<1QQR`bHb>G(_WBe%kz?_j}l zlymL|Z-irG*II>Zbqn*Mj?nPS83c=(!&!6NvxJ8?pKw_Z#sJl{62;E-nc#KC)oWc~ zH)*8OMaIJW{LH$sxqW^2NHYcgARbt7<&=Wl)9)}%?fTYq7 zBT{kYb^V6QW1)GT!7s;b8jA9(pH?-*X*Mf!3Es3>pT71P-;Rhrn_{+oZs=^u7rmS> z6AL$BF%;!5eBcG4s7afYp$MCGZ|9W&p6)ivHfL_lni=rcT^)3hh7-*u4?YxUv2%WL zJM3{e)iv&LxEG8tNI`K7@1w>WlFn*#p+l%_|EUJ`Y%_$ZrO^!avy z?BHv>Ke4a{=K%BfqQhV=WjWkviZuMG*~Iz1eF+oSAdxtz!rC^1)KXH)fg7zCIQQqA zj*&sM1BKkXCT5U7*tD~`dmHq|*{-x>BD=a{L)mq&m(;^=q-`a<{Zm(S2jgU#4Ym!J zDM_YvoTBFlGoddev;{X$t(n{+eNv;Kb8DxZ!WP7_?z%rW(2bWEAhf{3ImeCFC%(0P zG6*#2M>QoQLlr+`d>P;&r2=%`g;oKz8N&hXTzm2 zoR~Kn^2VTnj|W*kWXQ042~f|Nqs!4CnPiAsGX|5O;jh96_$8X$?fV{yy}U59%$FlD z;v|i8d{Kkk#Gd#b5rW$7wLux0P`b;+P9xk=LflbQ3+|*5Ijai$MvCq0Z6NsUZzpA5j_uHc@QEDTZHgWsFVFLXU`= zC>A(n%x0WMYv2DuzFe(RQ?pNGq4Dxe&Dz1zXKC9~Mnaj?yY&+fU9VH5MpqZ*mzsfI z%Tr`N^OC5%iMBEAI%uOF6-j75i*3{BQwWGi&KDd?*+8SUn=Sx|XYK?cG_p@U6O=e0 zC|M4bo5pLrl+D6pkqLEGcLti>T_99B{aG*C>fs23;2iUrt~uk`-vEJ=*JNnGKF_*w z3qf!n2wGUQEt;LIz0-FgKf3%@X{O|?jwA_rPaHnsFpoZU(t)_`+#UM}UE#CE-JZPT zmN4YbWNg`Jr`}{gABd7yi)FRD#OaxGys)-~PNfaPQ4 zR_JA5(wJz>Fewmq@=#FJvvN% zjNZtc!)*Gknok|WVSN|Qklsa3(e{+h4@*{V3?J3Y@sv8sOAb1h0wpC~#maGU{lRaP zpb^*tL0LfiQ8*&HvNPBQYEmFYJBhiwxk2q?7%`lp2b*2o{fNC4d=RV())4txQ;ht#ji*XGlo#AE@{TVq4DP*8{XrnwLb(k7wI43eD=G> z$a!B>Mtqh*`RkWH2T;eVpt&a^bY+k|S4o}q17*iasj^QzRx@+C> zu099r{*5l&B7&YWX|}i)wDp0b0eq!FX*|g07+;kvOeJ)zD8f=wH^q6IWrJ1G}kN zj3Z^be9@l`_l`*Vqqu2BO-01QK4D{fcEB!+u#2{m?dN%pw6=gx{Ak;PSLu?P;{aCv zUA(rS=5_TuKznPFUx4F2r$9sD+`HN6?Q1wBd;|JmQ_F zQFdohXKyKL>#rT=^lNVbf&L=Jl*8Z-W>dKHdumQ;sj${^ zZdH8H5_QImjSB&*k$~d{JRkO3KoF54|EUd`#d@O{^lgFL(DGr-b2`=eqRd3?C^A)|fpC;2CI2EoAaxTq7XYN}_|$xWxcTv~`(w`fnG7 z{(jeBoP5m&&wZi}AeAi+4m0j_H;H<3wY~Bivl6U$N2(O|P_K=d9Pxo72TdXxv4!Cpodw=vL!72A0$Qby!hUA6f)#+~ z&ZRB5cfl-ertrhq<>`RC?Bo0s>$Tm3Y`TId*B&uX?c?>i^?IQ2nTSyKOy%N0W@(^H z?%HpH@*)0J7f5k8md1Oq&}q5}#bT+Frm9CN{XCD>9G-lH7A!4dm&hgn8uVz53lw*@ z?Lm0TgS^Y0@4;pIZ5hq3iIKcI!F%OAv>oK#mT6T+FBu#Yf9uyKF3rRl!;H#n(#dBK zH~t#tph7w0*ChN|E;`FPT$C-TMNGF=AZiVjYr#WE`eAb8JWxR$eLl5hssyj=R$_u+kW-9kh6P30caz> zivDMW>QKr?K&&Pfa9q=PJR5N!i}>)m%wzhx8|$#sC|JXKhzToHaDhnV4TkhNPTg{7 z5N;j45u*gFIj}H=yiE#6HxfSoxE-H7>oB7^{say}{8Ihl>EK?_jRN;FwC;i?s+x{8 zwr$X>Ndg$o0aG~$pItT&$2cgp>GjKY;{TI&Ew6ov!=8&BylCJ*6}Z02Ho!bz zx;S6}Fj^n*optCQUjp^ejG0=4-gZvuL0f0Vd85`4fXR8KyVI|;3bXASVv<)t{A9G( zxc%g`ufnoN;#aS;wMNSP%qedqF?ali2vXQsKULq^T`PR#_1lWSGVYTsxA9i4AQy@7 ze6!r5oU?wn8!Kj~wXW{{){7hsIpKH}kn!DAo|kh6J}0gRV7uyQQ`mtA(y?6Q;JoS3 z?hSjs=){t$qrYL!Sq@g zYO@m?)HO{X-&yIuEAea=r=RFv?@dt7jwm3#IyifTy=nhSn|{P6qczU>$TmJ{(I~4sBWOvZkaQA-dLS>d-j5|- z4^axwNBom4j9f5$l_jE7pS!*RX|CbR&{9Vwe4TS8=jr1TVv=sMK*#l$U21z&RX;9E z+bkmz-4<}wPwCe3atq!1<4@U*&>P`^q@z`T`Yy>{T|7X9xt)>OuzMODw8!_{#t!XV zS&LMTNIOHAhR=>qC2ED@$#w zBQ}LQWhN)X-Z&1PpSH?b_DC;xs@11QtJFL?!WX%m%~x1*h0QYk0%v7~&h$u!q~VkW zLTF!UT)JHNd-J_3<(Pxqw#rrUg`p6y%*}`2bz1H7ZCI0gr3AhAzGWbdV z488?KOA7kT;nM@9kMs4fuu_mSo9xH8%I#Q(ZNvo5-RVEZ*X!=OHxXi98-q!=kKm+< zn0{%eYMkd>VV;)5UFKmpWOM|*Iu~+AhM%e(EI+L|PT0b5SNrY0sP`dQ{<5+Q-^^Y$ zuy;+~tmJB9ZRp^nPY!w<|8gGHeW}$zEIEJT0tua-nO5)je2NI@nr?G<-eWp+wxO1i zj>x)tqKx=+)!$}3wL>@}RSuU{$Q_*f?H`fZnF$UJr=lumvl3_erEeBRuNOIGIt=d8 z)NAFeo8$wY*G)Gp=r@yN`0!v0`t{_5iBS?;pYfEEFYYR*XXXMS6urXkp_bZE+uOP| z2{W5LX@s*KFBg+k>yb7Gr))g;oIh|U4r`1^|EMQkW1CHl=$zh^ups#>xH_~fxGO?o zkQ)}^Gj^+N=ei}P=&J4p#JFVtHNYLabC+UE2PNBKy?@ImevIxKsRkM&NODpUEY`LriMbNb}Lgqcp) z?Q#}1tdUUU&J>=Rv*GDj`@xjSl8s6piRcHsz&eZmb4?~S>c)2*{Ibm{4v9WPyX}3q zPX!I5!nHRbM#Not^a}m(z#UU941N;#A8wI*72B5@qYw-pBn;MNvG=AR3uA*=v4iE| zP13=L5;J~K?7oS5ZBwS{*d}CJYCaDaEG{}3UfpaO&%9Q?O7)u26J?&(yHEJ7#xx}hRlEBld{q@iT7>{59=?j&!#YM^a3@Bk8OtuLb zFKfQW&>ZU7xgqx|K7Ffn1LC@~J!w>2@?kJ0C_yn$UMb9m_S-*-Gb;gZ5kn1Y{qn3rh``0*?DbGB7qUWE zRv7uUrS%tPvOc}Q@;k-8c|r3g(ZCGd;$`< zjKKtOD|7UATs$zNJb0rOKCQBn&Zvg4_u#aGx#SGNGrH&!&S!wPD#@VS1 z)vM%0SUumetw0`b8FUFhqOu2m2M&%k!D2+Jr;n(8KxxD|QPLXUfUybhJi@c$clt%V zbAP^YsR(?T`b=DUIm5fu0Fq6kGpb+VI(2EPM1c*?fLFFVR7Y??#k8Hhl>w`BXGgp< zyE_)|3V!Gfz|G`|7L0<&5s8hw&x^5G#Ej;TNBW@ii;WU|731gEpU8%cs+6~FFxvgt zTxIdCO3kxxDhU8ct;Sx>FxK4G?(kFJYT(R9cD_4cXSvn?Mj-h+Oq=|{m%V_I$C#9pU}$WBEN-NRH1Q~lu`(j0X-qXyLi3Aece{US!+sM zbF3fGI@Pus-B?tJXK#OBM+hAFCS^3(=b!*sbb=sD9gDgF08ecBx~}qT)kJ@6qtnzkhO_@RHKscbd(1lc78nuc?BSA%HFi9Je`D@#WyE^ z4AftHf6cVL*Lf9_4MmA= z1YFQ^%{hIvT3cWNwUiy4HAlV!(EdNZ@%p-7`)SVB9!x}TbF!@E5K%4Zl%}(4x8A*o zWR_zC65kG&)8JMr3Xg*_(yFFTo>I7xLLE&Kr(07c-+vMtzFVm(OIW93?8zYax|kBb z-cfx5w%jB$hQq}{${iL$dlvIZTYrv#!!K*&QtIqJsx-gjc@8I?C0}UN)841pf(R~w z%YUvb{3~&jZaE?@C)&%qGlA!F&4Pluf9(!7l2S}pi>_x3Eru!Bg$Evfwkr@nT{V4< zR%-nY%Qe4d8d`I}{*9GUe2jMY^I#|H2nTw|~U>hEd%M-$iXd1CXr$%^$) z*bREDJIFkEmu#a}!Wzw@_)L|WWAzg4=g?wAp^?DDzrZ&`F{2wEKb;F7rJC#XU|$VY z_coAZdTbth*~j3Ijnt4&kY24CiJx=M$x*`0_1xGs&U+y6E_-%lDaZm;u5TIqHWPxhnl!}V@Qli<BhxhaPS9&2Y+9lr@}_2mlkkHPp7VN9 z>{GTh?MQAca|+P}_PL4u90n>^0|8N9QYg5UUFb|&ZXw>F`K{y>9qaW6e~%q8;`GP8 z#vFqZiNR}r)mQDfh%<#Rxq-d;S7x#~yJ!)ZZjwVdK5l(_3O)ABl%~tyRmGOEMpEY#;1%aC@YH9AmjzduL(rJDFa&(pY<;rv>)KQ@n_^wa8X>Pcj_ z{5sj$wU;9F@V_Zz zaUT1f8I;ns?;b9SgRM>Hx!8U@i9$)Z z7gl1QPW)l3q_uyRO(L>=z89Mrvw&ikD|)tzpM9>Pg;`hjDiTqWlCaRs1L}1aO%WJL z6r#4on~VCKAKDP!RiWpXxNg;4&F(z6l6%69ETse`wPLQ1OF8-U;NV(>J zUb6D5T;U~v)kRSq>Kz4~{GK^Xt;+E1 z9&~5buYip%56#|JQABy!QICpy27*o2;b#gf`0l0dpPZ$^nApJat^6~ztGUqlN(x$C z36+lkx+_=wl~@UMf1-vSg7gI=ncU(WQZ>l_fhMTD44NoC;aX~h*)~CypB(BM0%)dh zkwHgm>hOs9__x$z8Qi1Dq)b_Po`?>Y-S?)=?rv;ycJ zj^^I0NYc)`J7(24qPwMXbagn=OU^P+(ToZWNh*$oexGA-=N?~kFv=!ywd6TgVJD-x z!{vZ|+<=s-%4jSlG1B2lqh3hci=t|+iALe5u2XN_$uL9K;10edgSU6pcxWBxDwq`7 z{;)3Oua_|x)7JX=er?VUrQzI=H8Jus*YI&|JHzx>{O2h_v+~;I%ySW&bSy^|D zgwLk)M_1QN5vxBB%ZcpSv7nc=>DOcDa=L>1+BsDBy6EU$AAijMG!XB&>nh2Q(eAF( z8|un7CG1JB-=Q#BBr((7v|KWTs;ahVs(*yr)LLNTf+eFyqBi6>smq#aUVE^aS7FVn zKaf8VWD6p5QBvuot0s-&HIT#`h|7DfDGmRbbew6Eu21?3M8d3&wt5#ju|s{#G_7E4 zQ;q>c@#8Q-=Ox!qsTz|M#>Ru2EB1~0!)W7j{w;g?NqIlYoIp3E1xRCPt(xWvOAuBH zWW#C6M?1xWP7He+;I}u)53Yy!*SApn3(71?@$;SP``jl_D*HV?#j~Zl)y&9qllM=o zQREV(OrK*L=|=-|3)x4rXniJd}9 z5chl8H2K`Ym7p#98%X5GfQej?>7qvB0MwTTn2h53FH9hE4m{lIHE-14g zUc;5+nT>?)v7nE`LPK7%%0PPAL0SRKRw}m&KfE^2@%P6dE;;EU1V3*eb)aRz(`9MZ z#6xdGVmW(VyHwfSh#L6k3i2Uj*_5$*@KAwn8}x{;o7Zw@b*eP)mT#c$70TQBFiOx* z21RtxLD`uT`x9m!kS?5{FEV{-m^k~bgbfAzSi@B)@Ry=|N{P-?#9!IP=SbSxXYvKf z3p-Cw8iv+nm&w__Yy;L;p23Bw9p;mSjmEXM)4y%Y2~sT0FLF8w+JFESSHIWT^7{T| z(ggC`VDXPrtI;^19*|kTboVg+>n2c9k%l^M1mczD7E-O2_1|ESvD0YQ+882ramsHUQ z-4;eGix&kUq4?xTeO2;ERlVTJef_io)y2oa1gIO3$GEhu(%n>jedC2XlRqPRX{1y%gcAzhIA#gM|&*y;S~z~<01+s})fQ-GJDs^O}1n#rfLY{_<# zuT@?g&%hV^ag2}uFXNJAjvSD4Pc>imEcYigPuAlV)S z55c)-1yNcX&EtN4;*8g?6| zM6^yUT+0@&mmYes>kj}}uyRG}|3)A0|H1XU|MXjSeIXJD3FyULGF2zy8E2Cscmhr% zCH`a9#liJbs!fCAi+4=HCDREQqN(JXZ%4;CMj3(t4!%t#`br2DA<&NbF#)-S{akz~ zIb--#Uq-`KRN1%c+^0P=5lM|?sgLFOz8{O=bt_;EL=ixX*6C<A z*+K41+*wY7CzXqpc-z%XVx%L~RnZ z*fm>mUZ*D%RwQU8pf0#nA3+|Mb-!Y>SU>da+d+7I9d)$NP-~CtP1E;9LCXZtYs=@t z%NKIpci(yA>TzLf*#^f?0Dvr%;hOLErtF;)X(Z?o^m`$gDl0cLC=m*MjN>+h4EL(F zZ)IjDB#r}y-8G!02r*YpZn|@J8UwJp>~M`i?#c%e!$w86p_umt_}UW3Y@g0E`PkE_ zZ4^5cib9Dh%T0?1vZ4jLN;7UW1ulat5xV>Fopy$=u~*3P*4cJhH|NMMxtURZ)lJNk z>Coe=@jh#3j?(B|xF0Soz$Td#z9SZtJ6tlq%E(ndW>M7(U37tRKbhVuxfrs~#XUk{ zDe(kNm-w=g)B~~`k$U-F#BocDXQI(>Ag>#tlO9b#>g1@7rOzKWh=q=o&tQC^LXCLD z6%9`XU)Q82m)=_jiN)ghz9;_bzsmy!nFkx_=A{*p~#yszKH|0wL zD}o=ScqQ`8jtM=bi{oGY2JxmhX>TF4@hpjvVT58_9O}J6;XQ8`yQNS4$xGq;Qyl^? zg3fo362Lg*`}S*%QQeo~N`eUI<}&c4H-dE27SB14pSTSr&z`Jv3#j>cK(abxRJ4fd z5xZG0p`jSk9TaSEjTH5HS^2R=MVd?acA4<+MIu0q)$kwA7-(@)wZ4fHxM8`V;Uh?y zS>$*xX*b0qKBY91D>l!HOzX;=Wpp-xA0wy3E=z9;E)rH252edZJKabN@I*P1ssv-k zOHb7-rq@da9%{EifLGvV3WKF5rZ?p|{-v4wo&#*iiHpOA4tG zhgbUq`c0WpUk{Q|^1G!ro&mYa1#K^GtCa^$Ri85}=_lv%uVLqM$lB@H20iN4p#8(3 z@?`1@SUf|6mdHDGLO~O2<+wD)*yTfBw>7Fy=}c6gUT7-Xp5bmbKyCtuP6~R*k_#x* zO{eGuJ6u*n8x!IzmW}2?ZE=|Rz#>6UGU$U*PsdD`5ZC*Q@V`@G zG(bQj16^+(KVo}dX*Y1b4RQ0qAna_%{> z2do;?x8$8DFIWCfhDq^h(j_QkhmYoUJ5+BSj@kN<{3c@R$@C_jF~Kf-U;YQGkQ}mZ zJ|Z={jR+p>bZ>J+%14%|j6eypF5ggGFYn^vNd1MQ^T0ANi;iZ$vwX;|C)<2#~nYM6JhFdEO6` zDQCjv8Sf4^;#!rN&Rf;si*n0Z@Z2CBNI(u-M3YGP~ zY~rH#@wSq=f{&v5?`7uD9XXf~;`Z-@m{SL~F;cD91nNww`{ZSr4bXm-8nU2>w2slt98MAz#%hwDe=yGe z76F@-IqS$m?aV2SktmfuSQ-_S)Gi$y+_4CRFf0x8hf&mNIFnD9H`W+c`fyI^K}+n9 zf&>3ttpySJEty=i{dcE@5l@L(J>+cR+q-MI!&w&Y(6U;7s_IRY}^EGFk z6ezo`v&&vrB;mG{>>nKw?<+`bDpNC(`D&2+MMvm9ko>L0?7|>%6^r%B-X83u;n!wA zx%Ic({=s;Myi1?7?0w~f4)X~}u$w!AzS#ajDEtB=|=wf z5FyMhRk!2N_IybXX!RVpcE|S&H4}D`QH0C%gobMq%8ufaBpTwqjdGYc_(C8i6egVt zeX5klIrt1NH|y2+U&lHV?W263n0%M6(>0wP$^cjihuooLov&8_x$HW(Z_{mIZZC~D zU(@_CO}icNUgf?o7JNoQa+6JiK$gA*>NVvD4T`QBbD7uo=L{S6TYN)(b~zaPW&y|F zQyHxHac7r>V}=jXeG6Z~rfY4Q10||0H@qGBnfaN@w8WCqbBcxT&aH_rHpc^{6;=l9 z1%@G7H~{9DZZ<+LRu(FKSHzA6nK zj{@@^B$5TX-}KPZ4F2fx%Q0;~BKN#^kMPohV$WjSHca_0w0^h(dVL#lDU1lMX6=|c z#M1ALqBo``l+Kaionf{4We??k`>ALYrCKKGFh>#ZrMxAH|3z5yeiW-V>umIRX2?*IXVzYC;xYkBViL z!7jj(iQ9s9o2l-xnOh_%bRSWmB^23GnRPTT)-{V))w`;53&c-gk3B)`Xjmt8Ui;6+ zi*=d4F0lm@Icr@qh-oCZ30x9GOZ1S}z=FGsBLm{#v+n}o1J;4)ttI#vF0S`y+2<0F z@!q8e#Bd)7_~n9-^VPvu;;(?>;kUvwtv9h&t7}&wVw5?@3+z>C>_LR zQ$IM>Aaz-`#@)7UYw&K?Qu%W3d~#OrR?AlR-tI_qdqY5vj6X2Zzw^gU1TyCx`D3Ix z!O}ntS5Yx^I0e6{$RYRM0HA{t@MYnW6YO#y%7kStzte;9%CDKH7z-%iA z&920B0Rex<4|~V2)1dy%48yD*{@<^_KcSZvmb-tf2=ZMholxnPomz7{HWoVpYOuM^ zr)z@K|Lg9GY$RM6O73O78Y%7> z_a8z^1eUp2m-RXqALTD&lWmc~&(s%hnwXC|Sx$yjJ;(@IIZO1+CvA}ghC;uwbed6EoeV_Cj#t6YD@oFm{d$vBO;4(c z=MKBk7nR5Vt+$}JEMWLQ+YqRCER)7i;Ptwd1UkeES;vdJ5{!TwKeHph z7B2P@U+>xZw`F+n=kF5G{#8%gw5d;$Wruw1lJe^bm(*!@6wBEb%9G%8#c5ngnTCVs z6g94FJcRT&xJCcio{dl5?-68w{q=vxU&4>59_inSLjITR--hV!?+H!**E4P7oBrpo zfPW8=|Bt&erLq%!G4sa%jhmBOvEcnTi8?vszkvk%=l|g9x2(he<7WKgPW(Im=ZO6O zGF-JkCa3~50L;NKGspgW%xsha_63`uG;~B^u<(s0pmMbJ#noQ|X0L$IzQTJklN=M^ zc2&%UV^Y>9`gkuS(Xw4VuE-6Y<(Q&R*?i+yQ=I4i?Guyp4M8V<^i+T;sf84bV4~xT zfLtU*>KIpeE?lJ#5}E!Z$Vvq6=OVHWjvu@?U^(WkCDWuN{ODFJxsMwh0eFkE^IqL! zNm*HHOC}7We6QF0U&HTh-p_<^jJ3MMCy@qC-R(iq*`wLi+9Y3JL8c(O*6tZ*kbj}& z++``G$GBbj#P@2LW3K<`!6hc#hv3gRe94)tvE$X2O0Mz>$Cy0_7j1&0)@4p}rIEOt{ov~=s8`BBdVHI15+t=>)m{U7O|+hm#hP7Abgq6 zp98@x>G^Qnl05SGT3%a=?o;GOjhS4UF8l`p+qPvxa3}jY@hG|nc3-nxer=7%g2;I2 zCyfM&e)R6_cFmo8>9!DeyiaK1Byj+rPWIuZnxvAMQ{@LFkAe76>=Wz3sFx_Ua@nY9 zz4Wr>EeFjrz*Yk)CB4x-(S?(~coGD&#I;|x;h$N#(65TId+Y|v0wl06+u2kc#;Dh; zHGT4IX-9PqtK{PqnTMW(uL_0@St6*Y~O_XQ0ykNz__?AlCfGioW$q)xE_lzNk zvNoN8zQvolA%;E+m2T;<$XC)@8r-URxAt-5Ip1^JiCF^fM5&-AIlGGZ zIb7j7xk4NJtu% z5HO)dW8Svo-}QsQkE~L@M#x3JD(U9k4;nJM!NlI=Kk8LhP%$aSH%==sLo8loHf zvUED(AYo3)XWk=Vx+Ua|iP(<~U>+RYDEg;B4ttiTkDOmVNLR%=$f|ZKnATnwsKyIz z%Lf&SDr|tx%~0&Jo3;Xxr!0gM4soygwWl0|KZ5O!h9on6qR}&qUV9KZhVPT6#!W{n zuiJ-r!&R6{>B~qE-?3xC^4Rt)bsNSaUZJSKmw7QgiMeBvIl#M5_e1~zkZ<01Sy7AA zmELwreGn5+UN5+RxfCrPvoL>g?dt{^&l2T*kMnY>uj%WI>bW3)h7Gu}&%2wed3GN; zwOkXnAT*8%+E$8WB#!l61$F^TWpbe7y4}HB@+1qdi1+lyL#*h@(`|f>)n-o(3NCL3 z>iOeqQk5PJkEZ4?j?+f$U08BxIAuZ%=G2F$E#hMw2ac!a%<@u#oznFD+?W=$({D_| z_57-3d}m4-;4=yVf{x3w`-yd@+)0{u1<7bSDsd5akV%}qBqyM0ZG(liFcWm8$%^g? zXmh~T)^2^_3Z@NB4%BGlcFl*Ol$Q3>ta!?@EpN#4t6VMGX_hh`2#x2bli}zxMxJgq zqttjK*bm%;!#p}HJO#1ok?eyu_=TWmzsR9UNxEv}6feWRTPqKi9+x}B@i9m>G~9{C zd{hz>HC6lS`{saJIaA8B-xCNZc{4Y25$kxC7*)Hus!yF_NNpmHDh6)#M@=M3zc zn~^@tkB?d*vTN_{eZY2d3Jdy(+9$qHLTu5p{NY;VY%f_r((|+rQq3 zz~s0M2_9Ku0d#-p7x(i#>NI^m&WF->j4mFqzCn(}khUH5ObjPN&d{!{q$>K%sJ&*# ztbf20>AXj3YYtzNd3Hy-zxy}$qRC%YlF0jMPTI=7nSy1ubIxJO6Fs6o89R+fs{CAZ zCM?9k(5MApNHJFGa~sUR#cn)IKM9gkoH~PPlPn*(O}mwmMAD#RJ7tngdwj(q2}E_u z&HdWjDVhNwHw1wQ3Uni`1Pbhr6Z#frUK4$m(>h6g*M%|_x$jO#Z!x&&AUacwOa#Qlcmsdn%cvR(E zp!ak(D-uJnIuXCV-3VcOkDW53$0xjo$g8aFwWnjetceH7r@|B2R&FJ1f?taMHxWEK zAY_19ho4dnkrX$Hf<{%3jlW;AZb>}j)A=r+H_|lP9wE2BuhlQ+E2~M@^e{!5xWjP& z!{W=b4)+n`WzkDF^Ol}_K~Ca*J<_z{HD|ECMjwQ$ftst8up$K$sIdH!R@2C8{sg9$ zG(-OC{?wikYf_4pzXNvS#D){<>E$ySZQn~Lwb@-S%D>apUoAU%XZ`|E0=xzB`OueJPU@@uUEXmC{!QMAf{$9KrKhORwZ4+f z(!aZxJU3LBvi8{3f!zl9w$TXDOloc41NPy0oj9h7Fq0L%oZX>J^BvRvdGI&yN6@~Z z+7%xuyHs(e>rrZ*t)?qzMDd~h=pETV7P*9NF!aM=2hFp-Y-`PjQ8BV*Q#V1oz%8&E zoJ4qrj##1v$LaT zOhj_5QIT1Ao4!XceXRfIEw(j%aKW$N))-|Zdna7t)rz>|fq7lJO-!VEilr{l(6xR( z)LgLHjp`m=vC;CKRFck~N6-$)Sftwa*>&S?@W(^&d9;iSG41)V?gnZ}bRFSspdNB$ zrKe;+PF$BVk%*Hek9oNZGn=Z9`8;u6_3y%Uwv>4tDUah+HC7h4W@`?KEDU;Q(!J2| z27HHXg*1r1H&A^=scWHZ38dOII3I1H{-JX5(sEhjxJ8G%rzTi4yzPfj-TN#u5i7x~ zFi_@cCiAYe{(M+AFufVR#w~6s}j!_rMpC-#z z3bnUaw0)aNePqpLc0@&9Dq}YQN=rHcmcVG?Np_3JC)M98FZKVW*>+?@v+bmL^K-|# zyJ{O(n}%AkasziftqR56BC@D@0b|p9G0vlH?C@QfmTilsZiY=YH=#C$t z|8*1S2|Xn6*YiuO-j~m@LKVRk%!+#tg3=Z*KnXh)^@xs0S-`eOGtWdztEYJt_efR@ z-MzF!e8Gwet_Tg`f#Lf-odaYbLNiv%(shoFHP0C7w3^m%Wj(N!-W85W5%ts4hFQeUYKGf zs*kN+PU~H(Ytnoez4cCg4DcQF8{P-AUW%k>mZ|4j6Tsd;M_;?lRmA|SSAKWG)}mJ& zd@KCnS`|xc@zf-wo@?3X0T-+6L>^6Js_5Dv1OAGaf37`N-9gXB^F7R4@F;5N!@4rr zv7gsjcNEezK0u}7?*^5qn=gL=6fKW#4q;W%jjWv(uG3-5L(9-7B4A-^Z z@$+jiWNpb|`d68HiAld6huE!NcJNq&&`3}cZA#}EAZ%}}nDI7%v-co+bAN<_=_O)C z&P^YjCp`@fPu6ewM*KqQAHPB7LR`O7dd|80j7E6k4Ln9e$eJmoLeC1k^>QX5fUF|W zsTQjRo+-u6mC2&7rzW4MH#Z6|AWE&_3(C41QW2Ej>vr+C#M`#vUlMO0{qEVSjZZ`B zUKdIGbzLl3HksHTxL04kg#|%~i}xvvDVEMa5aEqk;Z9$72h41Ful|(H6COU)E zr4yMS)DzLjYiv4%-ustRC0bly85$c{_&mHnqa1of@Q{EmF4^jxeSz*$kt^qc-??w& zVyo@0D7%>jQC{*`lVY{ZE2oy5c18zWJHL zdpPTj?wyB(-hac!lKtGkJFVngr(#diV%CO_9e8J^MgJ_hmH*h!X}r4B$18k^?FhP- z=HYw`VKBb6=g{Ect-LUA>uJv>$MkHkv@-CL{gVDh*>2##l)VeM?-V4@+pb9fOLh!j zNUPI8`**@1YomM;hIU|n-%KL&g+VL3XXJ!)5!^WsTi>aAI;Lg%O>9zmg|2(Jx}YY; z$A8A8g8HJQ(}R}37hDZWu^0*}Uvu^jtnDszcM; zX=_tnLw(mO5tGDp=h1{bxA5%ywj`c?jZJu|1goe!A|2o;&ZOTp z9ZvnLSBuA9wU7DiN{!ITrYQ?Cut!W;iPr$p*nc28GDdigJ!XGW=Q{Bkyy)B;embj<*^Ps1)Aw1< z6-CD!#QVZ;v2h zTE~<;rgmsD5Wj7L>OCfr|}E)7qIEHa{q4% zCcpU>04Z&-IR_FbK|##DyP{GfCidwHc|OyR(Su>eX&*N`(>+N$@&!9d5~&dQ(U)6i z$rBWoF$Uo(sJV3YRCTK$8-d$##eov7syPJ*sO-d<>>VGyd||$g*=WD=LrDe&`2CJR z?c(>eqheTuXI5XJaQ$#j>o+@ppN9@6v^USh%<;y(mF6h%i*LdrUs_Oy^=ET&FZ_1};FXHq_CS#sd~G`(3vEcE%**g)rX0?C4`}Ngn0T{sz71 z$4zU6#o9Y%p3|zm9*2eCPQLld+skqOY-fG_NYwJc_4v*#>&tW~$LL=4)IeE%@<)^3UzmTh z6aPcq|39ev>+8HWHS>qEJHO;l0n<9>?ln!4E^i&%`eQRcbM~j0;IrPlBpI{U2O!L> z+%q=G>tcH}t~fmj|3gJ7z73`ElLFnk?kv4B%l{Plb8nTI)5xBU8u)?3a{rQ4|6cQ- z`hE4)QgcZWOD}kVaoB`CRLZus5ctn}))s~{U}cYyV#N!~Uo-wPd}{iyWXazX<-`Pb z)GeaU*9sj&8l8lFBmAnrOmGrwLC!yGSX%*4%txjCUjCZ#z~jM=|D>NbZTgFOcYWIN z6IiD67>#aLZF|wlM0#$5@OT|79u|cBX*%~}KvBc**NoX8+6oPbbrF}zqT&Dy(b_VT ziK#&Qg6omB!gxT&_t@%wB8Ri;z0YTYESA7uruY_uW0%o@bCfqMCOZgj*Ks`ejb;yd znMhmmtX*GO;S1=Ui_DKo4w9X0^OF)&={id5Y2Ccl9fo>yFo*^JYz*2!N@PiCTfoG8 zafeN6(W13&7#=ZpC6wK|d_NH4z?VA!jhZiDvzlNd6J3PBD9gA$;4V06K7Qhb>=B52 zm)wJqn#bpANP9cv9<=a+NT!?mnz5?|Uor>{kSxYybh8IbXfGNVkEfIye*lp*&0*hL zILGEi6;rmX#||@shc@7#I3PIfjBRc&X0^??#Op0CsCe@a$GFZA|GtiMan01a*OdF- z>!+Y>LHx*6M^le>*+y5&69DV!kf2>AvxXVJHWD$#!PFzeRwx}Q_7k5zSLP|C`} zjc!%9(zbntIUJkw*gb4$2v$J)Oih@p9VD)tz5p$s^uOU)bI@3@sLH!_0mJsIlK%%S zi;p7bX724lAe9WKheT$()Ze+u@(fIg#j;_c{^!)cwGwE%VCrg`mspn1N>0k}EOC3h zcG9HpbsoA(3;(5pq8m|cUDY}itbcw_q#x{CrFZny!spihKvA%pL~NK=RR`LfMxVgt z^rJb-e11flO!gn-Udi64Iy=WJ_wdTn1Qyi!y!LxDhlUy)I1vr~+?dJ$#C8F8>(9m4 zoJJ-L?ECas7;y+aMv<)(C_Si#g@I?m==6FiW7(QOgHm4FG356EeU0uEgPv6tj1~x^ zJ(B{&9nP z6EuF~eE6u&Z?1i5`alJI<&y~M3E!1(=gJsQ03kP$9eoev_7FN~yMf|~0?4OFrF zy<9@O-yyfPLv9-i<3Wr_Ns^d%)5QIJs=NdwUWp^VDM0&JL`rhWvDYOK`>{GVZ>W&E0^gC z#@(5fCw&Jam2^*XXkwv0i#GBlrjV(sm@c4ut$N{7lmI+CK`d5s$!`2}uRnvd-mX>p ze)=FKi~S*WSltx^$B0}n~Z>9%V1n+xt;DGetA zNy^i@RA<2%5UmLK!d0W^4kx+thNE?K`ZtqirK?#SBNCr;${F}h+tJYsV&~~Ra>M%y za#*lRYjh_|9fm~1F*r*Hj@b9iL-rOxr1V(4d}(xvHP^7SYQDI~HrIr@r&?1HC>09d zf2HVFLsRg9#SjJ6?~~(L*$~T`GuE1@CIhZlLK2&bn#LzL_i|NF==g|5haG``6m?M& zAD!oTN(`A&!6P^NJjRtAloJS?^VL+UU&~s*ISmz9{JnZ-^(+r^cdnwC4qsaf;IH!sl9A=^%9kWW-MrS3p00r z-KH`jWZG3Fp|;^%Qo&u!wR+^*S`pArB~-+c46xwzblr9`$CT}28IC=M@D9G4_gGRG zN3aYd#&d}EX{yVZTDubk@Ikt=oE?HH;R!_uZud40O`X}GZ` z8Wj~9eU4Ft>JB@pNWZHFb5B0joKt{H@lFdLQOzo7N_4MBZvY1roqNPy#@Q|zP1=;0 zf|afABgF@>F4wfjf&DVbZ(f7e_Az4tj9#6#XwnDGEDk?qAq5p1oTv45EXKOk7-vV8 zPGz3H>3Xbj@Qul}+@WQ3MTGDE!d4+uhpNi|OdVO>!OP?(msr!_u2(DcO0AcvJ;oR0 zQwvL`8GkIBH@=)#)tW>0O|-XVCGj%VHF5C@fpP~xO`;eoZAlND$rMFB4XXa?Yr>46 zz9i~VktHLdw@?t#2}-BI>CpUyQy4yhHigv@qQ-~@TAwIz(O6Y)&kr1yv^FqGpytl@ z_0Bs%=!sP4nm<1W`$8ZW!Ips7v)6@l)RL7c)>)+l47V^=6QtX3P?+2^-~Y-ce&*2FYuE>Th_ylA`M=0 zpI6T!+brNMeaes4B!3cfcJC$1hrk1x1F@n6rte!lx`L-Cb6EQwtzNP2n^?@s2bo+{ z1@80B!0?@UCkoBaw8Qf6SC9&C= z4ZglX<+7`n5>{CAX_15RwJhI z^Ec-ySi%wg1QkXy1sMA;wiEj2mr1pp;LE#~Bp6AoiDJ*X$kqKz{Q?)pmfag~qfhA} zg{HxqLpP7<9T;#eguU%#+#Jjjdiv=)hE++DGueUn%&9tno?%_HJ`KkQ=4{^q3%tz! zt5NTPt4`jPihP4PhYMbA(=(dUhLm*YXEWBFz|K>}+GVTB-TLjOygIv}Qe<)NMWWTN zwM_alDM(8*Aa+_|<$I6EWaG9k@Lth(-3Zc-L&S=^tlTqZ$3lB)OMkL=o*jzrNT*ME ze%MK~<2GuUBQ=C_#EI#yP%{}lyq45T7r8sF=|h^AIWK8=!cufT^(KNu*(%rq8W+}V zE-J~PGhTEnhng3nwn47)JPNEJ*V_DxNv$N=^8D#CeoJj;`t9Xn6B@_hitnyH+|^)H z@0#A>c2?3LqNu%*RXXjPn z8u*T_Anz0WK0=XRZSdxL8;NLr`5UnAcmSN&xSK<)@KJ^M9l|WgM z+vXDG%;CoT{QF^&>|yw9Lf7%Bna%3!WT}FO7pC)o%U6(PSW4KIB+WfxQPYWCtGPk2Vo#7hylvSbA+KJN1D-rHjc(VT zHl38c?DNDN+44xY0l|-j#VobEy&+!xk`P-bk!GA}azEA5kx-1%xPe@jF=f{htPYEt zf)B{B`}Lh};xEYb@C&R2M@#lJs_dnm+nfYqV}Woo>aRI#v#-fLmvi=smGE2zkFF8U zdJWTGK_0#J$Q=_jX7L}>e|)*Q>&itdRT2e{R9~NsNz!nL9Cv5GdJ&_t)MR!*wbaBE z?g)9fiCW`bRZntYv=Z}K@tnL9tgI**NERu@9oaDtg6{gm&k5NxwOp(@rk_=xTCMi2 zHafZpukQ2X9Z3c0M*8bs{h1Hf=Lh;nTyLzrViyN|d+T~^>c*^t?EPH_odn`xz9k@{ z@nLPnDc+-N%X8J?;79?!i-o@yu`+W$aUwZV?+ zG$)qj!L09ttj(c8R``EmYF9Dk@p!QcwbZf{r*-ugP<yejq{MwrpMXXte6R~^cCOF67o^(VZ1Ti4&< z9S^x$a>8IvMlBfgBT>v|CWLZ)DP_|@h6-bBK5P8C0w|;|WU0yxiW(1qcrn%r3A;@8 z`vFr-;)x!p*ew)_Xt}{i|Cj1O(0KCh;lkFECyr-xCAL<5ey{gf81bYiUaI`WdUgB0 z7rbC*gIam_NYtVd;%e%LU78upn4Vjb6O@3JOK#a#Z-P6FYfg8k94wG8EOTD6GaOmi zGFJ8@+BU1r0x^aA7V+^~L-zRg*h`ys>M*Y)KQf1Y}PL{6)H=*0O z)OkqOt4l6kq7q0nZOas6vHB_q#vpKA-orak#_Z*+cTPx0wvA|KbN56Z_%PAGH>oAQ zw9qWXZLO*l78O?UO#+Ws*O*l_-mzfao1>#}SvWG`WG=V0O-lOdTN8S3srFy2u|O!Q z+Cy-Z4sLnNb3@b-^=LimpffIZG3AIHd{Gn|-e?WWhWS2gp)VHVdlClUmpo&|i2&VY zC$8(Uj5kxQ-O@J1P&|AoMVn8|;@E1&Dk(6m#khjf}8>#3;dY}ffjf5(4t z9|dhdUcoCsZKlX2vy78XbfP6VY_|>gL*J?ic^w_Zj`|2lvuoQxFR1*Hs{wHu0&DgibmajKiEanLWMihdL|b&k)I`P0jHR}Q-iqxVA@BDF1_ z@XHNq{~M`$ef*9|Tg_T()cmy|`=^iJDb^0D``HriUf}Zn%o$1P?wuft_b~!)I$HP2 zT0{F(uj!JXnb+QOFJb$n^!`bM?yG#aqn)VH)Pk62-BEn^Fqg?C)pBL+Qk~9hj%Hyv zd1yPU^HL7vYw2=u@zq&OyL1FWL~